From d2226cc067aebd61113c584ccf833d55cf227a2d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Wed, 1 Dec 2021 22:15:31 +0200 Subject: [PATCH 001/132] Tear down the old filtering system Tests and dependent functionality in other extensions will be re-added later on. --- bot/bot.py | 25 - bot/converters.py | 48 -- bot/exts/filters/__init__.py | 0 bot/exts/filters/antimalware.py | 106 --- bot/exts/filters/antispam.py | 324 -------- bot/exts/filters/filter_lists.py | 297 ------- bot/exts/filters/filtering.py | 735 ------------------ bot/exts/filters/security.py | 30 - bot/exts/filters/token_remover.py | 233 ------ bot/exts/filters/webhook_remover.py | 94 --- bot/exts/info/codeblock/_cog.py | 4 - .../moderation/watchchannels/_watchchannel.py | 6 +- bot/rules/__init__.py | 12 - bot/rules/attachments.py | 26 - bot/rules/burst.py | 23 - bot/rules/burst_shared.py | 18 - bot/rules/chars.py | 24 - bot/rules/discord_emojis.py | 34 - bot/rules/duplicates.py | 28 - bot/rules/links.py | 36 - bot/rules/mentions.py | 28 - bot/rules/newlines.py | 45 -- bot/rules/role_mentions.py | 24 - tests/bot/exts/filters/__init__.py | 0 tests/bot/exts/filters/test_antimalware.py | 202 ----- tests/bot/exts/filters/test_antispam.py | 35 - tests/bot/exts/filters/test_filtering.py | 40 - tests/bot/exts/filters/test_security.py | 53 -- tests/bot/exts/filters/test_token_remover.py | 409 ---------- tests/bot/rules/__init__.py | 76 -- tests/bot/rules/test_attachments.py | 69 -- tests/bot/rules/test_burst.py | 54 -- tests/bot/rules/test_burst_shared.py | 57 -- tests/bot/rules/test_chars.py | 64 -- tests/bot/rules/test_discord_emojis.py | 73 -- tests/bot/rules/test_duplicates.py | 64 -- tests/bot/rules/test_links.py | 67 -- tests/bot/rules/test_mentions.py | 83 -- tests/bot/rules/test_newlines.py | 102 --- tests/bot/rules/test_role_mentions.py | 55 -- 40 files changed, 1 insertion(+), 3702 deletions(-) delete mode 100644 bot/exts/filters/__init__.py delete mode 100644 bot/exts/filters/antimalware.py delete mode 100644 bot/exts/filters/antispam.py delete mode 100644 bot/exts/filters/filter_lists.py delete mode 100644 bot/exts/filters/filtering.py delete mode 100644 bot/exts/filters/security.py delete mode 100644 bot/exts/filters/token_remover.py delete mode 100644 bot/exts/filters/webhook_remover.py delete mode 100644 bot/rules/__init__.py delete mode 100644 bot/rules/attachments.py delete mode 100644 bot/rules/burst.py delete mode 100644 bot/rules/burst_shared.py delete mode 100644 bot/rules/chars.py delete mode 100644 bot/rules/discord_emojis.py delete mode 100644 bot/rules/duplicates.py delete mode 100644 bot/rules/links.py delete mode 100644 bot/rules/mentions.py delete mode 100644 bot/rules/newlines.py delete mode 100644 bot/rules/role_mentions.py delete mode 100644 tests/bot/exts/filters/__init__.py delete mode 100644 tests/bot/exts/filters/test_antimalware.py delete mode 100644 tests/bot/exts/filters/test_antispam.py delete mode 100644 tests/bot/exts/filters/test_filtering.py delete mode 100644 tests/bot/exts/filters/test_security.py delete mode 100644 tests/bot/exts/filters/test_token_remover.py delete mode 100644 tests/bot/rules/__init__.py delete mode 100644 tests/bot/rules/test_attachments.py delete mode 100644 tests/bot/rules/test_burst.py delete mode 100644 tests/bot/rules/test_burst_shared.py delete mode 100644 tests/bot/rules/test_chars.py delete mode 100644 tests/bot/rules/test_discord_emojis.py delete mode 100644 tests/bot/rules/test_duplicates.py delete mode 100644 tests/bot/rules/test_links.py delete mode 100644 tests/bot/rules/test_mentions.py delete mode 100644 tests/bot/rules/test_newlines.py delete mode 100644 tests/bot/rules/test_role_mentions.py diff --git a/bot/bot.py b/bot/bot.py index aff07cd32b..e40c3f8c1d 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -27,8 +27,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.filter_list_cache = defaultdict(dict) - async def ping_services(self) -> None: """A helper to make sure all the services the bot relies on are available on startup.""" # Connect Site/API @@ -45,33 +43,10 @@ async def ping_services(self) -> None: raise await asyncio.sleep(constants.URLs.connect_cooldown) - def insert_item_into_filter_list_cache(self, item: dict[str, str]) -> None: - """Add an item to the bots filter_list_cache.""" - type_ = item["type"] - allowed = item["allowed"] - content = item["content"] - - self.filter_list_cache[f"{type_}.{allowed}"][content] = { - "id": item["id"], - "comment": item["comment"], - "created_at": item["created_at"], - "updated_at": item["updated_at"], - } - - async def cache_filter_list_data(self) -> None: - """Cache all the data in the FilterList on the site.""" - full_cache = await self.api_client.get('bot/filter-lists') - - for item in full_cache: - self.insert_item_into_filter_list_cache(item) - async def setup_hook(self) -> None: """Default async initialisation method for discord.py.""" await super().setup_hook() - # Build the FilterList cache - await self.cache_filter_list_data() - # This is not awaited to avoid a deadlock with any cogs that have # wait_until_guild_available in their cog_load method. scheduling.create_task(self.load_extensions(exts)) diff --git a/bot/converters.py b/bot/converters.py index 5800ea0442..23bef0dccf 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -68,54 +68,6 @@ async def convert(self, ctx: Context, server_invite: str) -> dict: raise BadArgument("This does not appear to be a valid Discord server invite.") -class ValidFilterListType(Converter): - """ - A converter that checks whether the given string is a valid FilterList type. - - Raises `BadArgument` if the argument is not a valid FilterList type, and simply - passes through the given argument otherwise. - """ - - @staticmethod - async def get_valid_types(bot: Bot) -> list: - """ - Try to get a list of valid filter list types. - - Raise a BadArgument if the API can't respond. - """ - try: - valid_types = await bot.api_client.get('bot/filter-lists/get-types') - except ResponseCodeError: - raise BadArgument("Cannot validate list_type: Unable to fetch valid types from API.") - - return [enum for enum, classname in valid_types] - - async def convert(self, ctx: Context, list_type: str) -> str: - """Checks whether the given string is a valid FilterList type.""" - valid_types = await self.get_valid_types(ctx.bot) - list_type = list_type.upper() - - if list_type not in valid_types: - - # Maybe the user is using the plural form of this type, - # e.g. "guild_invites" instead of "guild_invite". - # - # This code will support the simple plural form (a single 's' at the end), - # which works for all current list types, but if a list type is added in the future - # which has an irregular plural form (like 'ies'), this code will need to be - # refactored to support this. - if list_type.endswith("S") and list_type[:-1] in valid_types: - list_type = list_type[:-1] - - else: - valid_types_list = '\n'.join([f"• {type_.lower()}" for type_ in valid_types]) - raise BadArgument( - f"You have provided an invalid list type!\n\n" - f"Please provide one of the following: \n{valid_types_list}" - ) - return list_type - - class Extension(Converter): """ Fully qualify the name of an extension and ensure it exists. diff --git a/bot/exts/filters/__init__.py b/bot/exts/filters/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/bot/exts/filters/antimalware.py b/bot/exts/filters/antimalware.py deleted file mode 100644 index ff39700a6f..0000000000 --- a/bot/exts/filters/antimalware.py +++ /dev/null @@ -1,106 +0,0 @@ -import typing as t -from os.path import splitext - -from discord import Embed, Message, NotFound -from discord.ext.commands import Cog - -from bot.bot import Bot -from bot.constants import Channels, Filter, URLs -from bot.exts.events.code_jams._channels import CATEGORY_NAME as JAM_CATEGORY_NAME -from bot.log import get_logger - -log = get_logger(__name__) - -PY_EMBED_DESCRIPTION = ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" -) - -TXT_LIKE_FILES = {".txt", ".csv", ".json"} -TXT_EMBED_DESCRIPTION = ( - "You either uploaded a `{blocked_extension}` file or entered a message that was too long. " - f"Please use our [paste bin]({URLs.site_schema}{URLs.site_paste}) instead." -) - -DISALLOWED_EMBED_DESCRIPTION = ( - "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " - "We currently allow the following file types: **{joined_whitelist}**.\n\n" - "Feel free to ask in {meta_channel_mention} if you think this is a mistake." -) - - -class AntiMalware(Cog): - """Delete messages which contain attachments with non-whitelisted file extensions.""" - - def __init__(self, bot: Bot): - self.bot = bot - - def _get_whitelisted_file_formats(self) -> list: - """Get the file formats currently on the whitelist.""" - return self.bot.filter_list_cache['FILE_FORMAT.True'].keys() - - def _get_disallowed_extensions(self, message: Message) -> t.Iterable[str]: - """Get an iterable containing all the disallowed extensions of attachments.""" - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} - extensions_blocked = file_extensions - set(self._get_whitelisted_file_formats()) - return extensions_blocked - - @Cog.listener() - async def on_message(self, message: Message) -> None: - """Identify messages with prohibited attachments.""" - # Return when message don't have attachment and don't moderate DMs - if not message.attachments or not message.guild: - return - - # Ignore webhook and bot messages - if message.webhook_id or message.author.bot: - return - - # Ignore code jam channels - if getattr(message.channel, "category", None) and message.channel.category.name == JAM_CATEGORY_NAME: - return - - # Check if user is staff, if is, return - # Since we only care that roles exist to iterate over, check for the attr rather than a User/Member instance - if hasattr(message.author, "roles") and any(role.id in Filter.role_whitelist for role in message.author.roles): - return - - embed = Embed() - extensions_blocked = self._get_disallowed_extensions(message) - blocked_extensions_str = ', '.join(extensions_blocked) - if ".py" in extensions_blocked: - # Short-circuit on *.py files to provide a pastebin link - embed.description = PY_EMBED_DESCRIPTION - elif extensions := TXT_LIKE_FILES.intersection(extensions_blocked): - # Work around Discord AutoConversion of messages longer than 2000 chars to .txt - cmd_channel = self.bot.get_channel(Channels.bot_commands) - embed.description = TXT_EMBED_DESCRIPTION.format( - blocked_extension=extensions.pop(), - cmd_channel_mention=cmd_channel.mention - ) - elif extensions_blocked: - meta_channel = self.bot.get_channel(Channels.meta) - embed.description = DISALLOWED_EMBED_DESCRIPTION.format( - joined_whitelist=', '.join(self._get_whitelisted_file_formats()), - blocked_extensions_str=blocked_extensions_str, - meta_channel_mention=meta_channel.mention, - ) - - if embed.description: - log.info( - f"User '{message.author}' ({message.author.id}) uploaded blacklisted file(s): {blocked_extensions_str}", - extra={"attachment_list": [attachment.filename for attachment in message.attachments]} - ) - - await message.channel.send(f"Hey {message.author.mention}!", embed=embed) - - # Delete the offending message: - try: - await message.delete() - except NotFound: - log.info(f"Tried to delete message `{message.id}`, but message could not be found.") - - -async def setup(bot: Bot) -> None: - """Load the AntiMalware cog.""" - await bot.add_cog(AntiMalware(bot)) diff --git a/bot/exts/filters/antispam.py b/bot/exts/filters/antispam.py deleted file mode 100644 index 3b925bacd6..0000000000 --- a/bot/exts/filters/antispam.py +++ /dev/null @@ -1,324 +0,0 @@ -import asyncio -from collections import defaultdict -from collections.abc import Mapping -from dataclasses import dataclass, field -from datetime import timedelta -from itertools import takewhile -from operator import attrgetter, itemgetter -from typing import Dict, Iterable, List, Set - -import arrow -from botcore.utils import scheduling -from discord import Colour, Member, Message, MessageType, NotFound, Object, TextChannel -from discord.ext.commands import Cog - -from bot import rules -from bot.bot import Bot -from bot.constants import ( - AntiSpam as AntiSpamConfig, Channels, Colours, DEBUG_MODE, Event, Filter, Guild as GuildConfig, Icons -) -from bot.converters import Duration -from bot.exts.events.code_jams._channels import CATEGORY_NAME as JAM_CATEGORY_NAME -from bot.exts.moderation.modlog import ModLog -from bot.log import get_logger -from bot.utils import lock -from bot.utils.message_cache import MessageCache -from bot.utils.messages import format_user, send_attachments - -log = get_logger(__name__) - -RULE_FUNCTION_MAPPING = { - 'attachments': rules.apply_attachments, - 'burst': rules.apply_burst, - # burst shared is temporarily disabled due to a bug - # 'burst_shared': rules.apply_burst_shared, - 'chars': rules.apply_chars, - 'discord_emojis': rules.apply_discord_emojis, - 'duplicates': rules.apply_duplicates, - 'links': rules.apply_links, - 'mentions': rules.apply_mentions, - 'newlines': rules.apply_newlines, - 'role_mentions': rules.apply_role_mentions, -} - - -@dataclass -class DeletionContext: - """Represents a Deletion Context for a single spam event.""" - - members: frozenset[Member] - triggered_in: TextChannel - channels: set[TextChannel] = field(default_factory=set) - rules: Set[str] = field(default_factory=set) - messages: Dict[int, Message] = field(default_factory=dict) - attachments: List[List[str]] = field(default_factory=list) - - async def add(self, rule_name: str, channels: Iterable[TextChannel], messages: Iterable[Message]) -> None: - """Adds new rule violation events to the deletion context.""" - self.rules.add(rule_name) - - self.channels.update(channels) - - for message in messages: - if message.id not in self.messages: - self.messages[message.id] = message - - # Re-upload attachments - destination = message.guild.get_channel(Channels.attachment_log) - urls = await send_attachments(message, destination, link_large=False) - self.attachments.append(urls) - - async def upload_messages(self, actor_id: int, modlog: ModLog) -> None: - """Method that takes care of uploading the queue and posting modlog alert.""" - triggered_by_users = ", ".join(format_user(m) for m in self.members) - triggered_in_channel = f"**Triggered in:** {self.triggered_in.mention}\n" if len(self.channels) > 1 else "" - channels_description = ", ".join(channel.mention for channel in self.channels) - - mod_alert_message = ( - f"**Triggered by:** {triggered_by_users}\n" - f"{triggered_in_channel}" - f"**Channels:** {channels_description}\n" - f"**Rules:** {', '.join(rule for rule in self.rules)}\n" - ) - - messages_as_list = list(self.messages.values()) - first_message = messages_as_list[0] - # For multiple messages and those with attachments or excessive newlines, use the logs API - if any(( - len(messages_as_list) > 1, - len(first_message.attachments) > 0, - first_message.content.count('\n') > 15 - )): - url = await modlog.upload_log(self.messages.values(), actor_id, self.attachments) - mod_alert_message += f"A complete log of the offending messages can be found [here]({url})" - else: - mod_alert_message += "Message:\n" - content = first_message.clean_content - remaining_chars = 4080 - len(mod_alert_message) - - if len(content) > remaining_chars: - url = await modlog.upload_log([first_message], actor_id, self.attachments) - log_site_msg = f"The full message can be found [here]({url})" - content = content[:remaining_chars - (3 + len(log_site_msg))] + "..." - - mod_alert_message += content - - await modlog.send_log_message( - content=", ".join(str(m.id) for m in self.members), # quality-of-life improvement for mobile moderators - icon_url=Icons.filtering, - colour=Colour(Colours.soft_red), - title="Spam detected!", - text=mod_alert_message, - thumbnail=first_message.author.display_avatar.url, - channel_id=Channels.mod_alerts, - ping_everyone=AntiSpamConfig.ping_everyone - ) - - -class AntiSpam(Cog): - """Cog that controls our anti-spam measures.""" - - def __init__(self, bot: Bot, validation_errors: Dict[str, str]) -> None: - self.bot = bot - self.validation_errors = validation_errors - role_id = AntiSpamConfig.punishment['role_id'] - self.muted_role = Object(role_id) - self.expiration_date_converter = Duration() - - self.message_deletion_queue = dict() - - # Fetch the rule configuration with the highest rule interval. - max_interval_config = max( - AntiSpamConfig.rules.values(), - key=itemgetter('interval') - ) - self.max_interval = max_interval_config['interval'] - self.cache = MessageCache(AntiSpamConfig.cache_size, newest_first=True) - - @property - def mod_log(self) -> ModLog: - """Allows for easy access of the ModLog cog.""" - return self.bot.get_cog("ModLog") - - async def cog_load(self) -> None: - """Unloads the cog and alerts admins if configuration validation failed.""" - await self.bot.wait_until_guild_available() - if self.validation_errors: - body = "**The following errors were encountered:**\n" - body += "\n".join(f"- {error}" for error in self.validation_errors.values()) - body += "\n\n**The cog has been unloaded.**" - - await self.mod_log.send_log_message( - title="Error: AntiSpam configuration validation failed!", - text=body, - ping_everyone=True, - icon_url=Icons.token_removed, - colour=Colour.red() - ) - - self.bot.remove_cog(self.__class__.__name__) - return - - @Cog.listener() - async def on_message(self, message: Message) -> None: - """Applies the antispam rules to each received message.""" - if ( - not message.guild - or message.guild.id != GuildConfig.id - or message.author.bot - or (getattr(message.channel, "category", None) and message.channel.category.name == JAM_CATEGORY_NAME) - or (message.channel.id in Filter.channel_whitelist and not DEBUG_MODE) - or (any(role.id in Filter.role_whitelist for role in message.author.roles) and not DEBUG_MODE) - or message.type == MessageType.auto_moderation_action - ): - return - - self.cache.append(message) - - earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.max_interval) - relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.cache)) - - for rule_name in AntiSpamConfig.rules: - rule_config = AntiSpamConfig.rules[rule_name] - rule_function = RULE_FUNCTION_MAPPING[rule_name] - - # Create a list of messages that were sent in the interval that the rule cares about. - latest_interesting_stamp = arrow.utcnow() - timedelta(seconds=rule_config['interval']) - messages_for_rule = list( - takewhile(lambda msg: msg.created_at > latest_interesting_stamp, relevant_messages) - ) - - result = await rule_function(message, messages_for_rule, rule_config) - - # If the rule returns `None`, that means the message didn't violate it. - # If it doesn't, it returns a tuple in the form `(str, Iterable[discord.Member])` - # which contains the reason for why the message violated the rule and - # an iterable of all members that violated the rule. - if result is not None: - self.bot.stats.incr(f"mod_alerts.{rule_name}") - reason, members, relevant_messages = result - full_reason = f"`{rule_name}` rule: {reason}" - - # If there's no spam event going on for this channel, start a new Message Deletion Context - authors_set = frozenset(members) - if authors_set not in self.message_deletion_queue: - log.trace(f"Creating queue for members `{authors_set}`") - self.message_deletion_queue[authors_set] = DeletionContext(authors_set, message.channel) - scheduling.create_task( - self._process_deletion_context(authors_set), - name=f"AntiSpam._process_deletion_context({authors_set})" - ) - - # Add the relevant of this trigger to the Deletion Context - await self.message_deletion_queue[authors_set].add( - rule_name=rule_name, - channels=set(message.channel for message in relevant_messages), - messages=relevant_messages - ) - - for member in members: - scheduling.create_task( - self.punish(message, member, full_reason), - name=f"AntiSpam.punish(message={message.id}, member={member.id}, rule={rule_name})" - ) - - await self.maybe_delete_messages(relevant_messages) - break - - @lock.lock_arg("antispam.punish", "member", attrgetter("id")) - async def punish(self, msg: Message, member: Member, reason: str) -> None: - """Punishes the given member for triggering an antispam rule.""" - if not any(role.id == self.muted_role.id for role in member.roles): - remove_role_after = AntiSpamConfig.punishment['remove_after'] - - # Get context and make sure the bot becomes the actor of infraction by patching the `author` attributes - context = await self.bot.get_context(msg) - context.author = self.bot.user - - # Since we're going to invoke the tempmute command directly, we need to manually call the converter. - dt_remove_role_after = await self.expiration_date_converter.convert(context, f"{remove_role_after}S") - await context.invoke( - self.bot.get_command('tempmute'), - member, - dt_remove_role_after, - reason=reason - ) - - async def maybe_delete_messages(self, messages: List[Message]) -> None: - """Cleans the messages if cleaning is configured.""" - if AntiSpamConfig.clean_offending: - # If we have more than one message, we can use bulk delete. - if len(messages) > 1: - message_ids = [message.id for message in messages] - self.mod_log.ignore(Event.message_delete, *message_ids) - channel_messages = defaultdict(list) - for message in messages: - channel_messages[message.channel].append(message) - for channel, messages in channel_messages.items(): - try: - await channel.delete_messages(messages) - except NotFound: - # In the rare case where we found messages matching the - # spam filter across multiple channels, it is possible - # that a single channel will only contain a single message - # to delete. If that should be the case, discord.py will - # use the "delete single message" endpoint instead of the - # bulk delete endpoint, and the single message deletion - # endpoint will complain if you give it that does not exist. - # As this means that we have no other message to delete in - # this channel (and message deletes work per-channel), - # we can just log an exception and carry on with business. - log.info(f"Tried to delete message `{messages[0].id}`, but message could not be found.") - - # Otherwise, the bulk delete endpoint will throw up. - # Delete the message directly instead. - else: - self.mod_log.ignore(Event.message_delete, messages[0].id) - try: - await messages[0].delete() - except NotFound: - log.info(f"Tried to delete message `{messages[0].id}`, but message could not be found.") - - async def _process_deletion_context(self, context_id: frozenset) -> None: - """Processes the Deletion Context queue.""" - log.trace("Sleeping before processing message deletion queue.") - await asyncio.sleep(10) - - if context_id not in self.message_deletion_queue: - log.error(f"Started processing deletion queue for context `{context_id}`, but it was not found!") - return - - deletion_context = self.message_deletion_queue.pop(context_id) - await deletion_context.upload_messages(self.bot.user.id, self.mod_log) - - @Cog.listener() - async def on_message_edit(self, before: Message, after: Message) -> None: - """Updates the message in the cache, if it's cached.""" - self.cache.update(after) - - -def validate_config(rules_: Mapping = AntiSpamConfig.rules) -> Dict[str, str]: - """Validates the antispam configs.""" - validation_errors = {} - for name, config in rules_.items(): - if name not in RULE_FUNCTION_MAPPING: - log.error( - f"Unrecognized antispam rule `{name}`. " - f"Valid rules are: {', '.join(RULE_FUNCTION_MAPPING)}" - ) - validation_errors[name] = f"`{name}` is not recognized as an antispam rule." - continue - for required_key in ('interval', 'max'): - if required_key not in config: - log.error( - f"`{required_key}` is required but was not " - f"set in rule `{name}`'s configuration." - ) - validation_errors[name] = f"Key `{required_key}` is required but not set for rule `{name}`" - return validation_errors - - -async def setup(bot: Bot) -> None: - """Validate the AntiSpam configs and load the AntiSpam cog.""" - validation_errors = validate_config() - await bot.add_cog(AntiSpam(bot, validation_errors)) diff --git a/bot/exts/filters/filter_lists.py b/bot/exts/filters/filter_lists.py deleted file mode 100644 index c643f9a849..0000000000 --- a/bot/exts/filters/filter_lists.py +++ /dev/null @@ -1,297 +0,0 @@ -import re -from typing import Optional - -from botcore.site_api import ResponseCodeError -from discord import Colour, Embed -from discord.ext.commands import BadArgument, Cog, Context, IDConverter, group, has_any_role - -from bot import constants -from bot.bot import Bot -from bot.constants import Channels -from bot.converters import ValidDiscordServerInvite, ValidFilterListType -from bot.log import get_logger -from bot.pagination import LinePaginator - -log = get_logger(__name__) - - -class FilterLists(Cog): - """Commands for blacklisting and whitelisting things.""" - - methods_with_filterlist_types = [ - "allow_add", - "allow_delete", - "allow_get", - "deny_add", - "deny_delete", - "deny_get", - ] - - def __init__(self, bot: Bot) -> None: - self.bot = bot - - async def cog_load(self) -> None: - """Add the valid FilterList types to the docstrings, so they'll appear in !help invocations.""" - await self.bot.wait_until_guild_available() - - # Add valid filterlist types to the docstrings - valid_types = await ValidFilterListType.get_valid_types(self.bot) - valid_types = [f"`{type_.lower()}`" for type_ in valid_types] - - for method_name in self.methods_with_filterlist_types: - command = getattr(self, method_name) - command.help = ( - f"{command.help}\n\nValid **list_type** values are {', '.join(valid_types)}." - ) - - async def _add_data( - self, - ctx: Context, - allowed: bool, - list_type: ValidFilterListType, - content: str, - comment: Optional[str] = None, - ) -> None: - """Add an item to a filterlist.""" - allow_type = "whitelist" if allowed else "blacklist" - - # If this is a guild invite, we gotta validate it. - if list_type == "GUILD_INVITE": - guild_data = await self._validate_guild_invite(ctx, content) - content = guild_data.get("id") - - # Some guild invites are autoban filters, which require the mod - # to set a comment which includes [autoban]. - # Having the guild name in the comment is still useful when reviewing - # filter list, so prepend it to the set comment in case some mod forgets. - guild_name_part = f'Guild "{guild_data["name"]}"' if "name" in guild_data else None - - comment = " - ".join( - comment_part - for comment_part in (guild_name_part, comment) - if comment_part - ) - - # If it's a file format, let's make sure it has a leading dot. - elif list_type == "FILE_FORMAT" and not content.startswith("."): - content = f".{content}" - - # If it's a filter token, validate the passed regex - elif list_type == "FILTER_TOKEN": - try: - re.compile(content) - except re.error as e: - await ctx.message.add_reaction("❌") - await ctx.send( - f"{ctx.author.mention} that's not a valid regex! " - f"Regex error message: {e.msg}." - ) - return - - # Try to add the item to the database - log.trace(f"Trying to add the {content} item to the {list_type} {allow_type}") - payload = { - "allowed": allowed, - "type": list_type, - "content": content, - "comment": comment, - } - - try: - item = await self.bot.api_client.post( - "bot/filter-lists", - json=payload - ) - except ResponseCodeError as e: - if e.status == 400: - await ctx.message.add_reaction("❌") - log.debug( - f"{ctx.author} tried to add data to a {allow_type}, but the API returned 400, " - "probably because the request violated the UniqueConstraint." - ) - raise BadArgument( - f"Unable to add the item to the {allow_type}. " - "The item probably already exists. Keep in mind that a " - "blacklist and a whitelist for the same item cannot co-exist, " - "and we do not permit any duplicates." - ) - raise - - # If it is an autoban trigger we send a warning in #mod-meta - if comment and "[autoban]" in comment: - await self.bot.get_channel(Channels.mod_meta).send( - f":warning: Heads-up! The new `{list_type}` filter " - f"`{content}` (`{comment}`) will automatically ban users." - ) - - # Insert the item into the cache - self.bot.insert_item_into_filter_list_cache(item) - await ctx.message.add_reaction("✅") - - async def _delete_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType, content: str) -> None: - """Remove an item from a filterlist.""" - allow_type = "whitelist" if allowed else "blacklist" - - # If this is a server invite, we need to convert it. - if list_type == "GUILD_INVITE" and not IDConverter()._get_id_match(content): - guild_data = await self._validate_guild_invite(ctx, content) - content = guild_data.get("id") - - # If it's a file format, let's make sure it has a leading dot. - elif list_type == "FILE_FORMAT" and not content.startswith("."): - content = f".{content}" - - # Find the content and delete it. - log.trace(f"Trying to delete the {content} item from the {list_type} {allow_type}") - item = self.bot.filter_list_cache[f"{list_type}.{allowed}"].get(content) - - if item is not None: - try: - await self.bot.api_client.delete( - f"bot/filter-lists/{item['id']}" - ) - del self.bot.filter_list_cache[f"{list_type}.{allowed}"][content] - await ctx.message.add_reaction("✅") - except ResponseCodeError as e: - log.debug( - f"{ctx.author} tried to delete an item with the id {item['id']}, but " - f"the API raised an unexpected error: {e}" - ) - await ctx.message.add_reaction("❌") - else: - await ctx.message.add_reaction("❌") - - async def _list_all_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType) -> None: - """Paginate and display all items in a filterlist.""" - allow_type = "whitelist" if allowed else "blacklist" - result = self.bot.filter_list_cache[f"{list_type}.{allowed}"] - - # Build a list of lines we want to show in the paginator - lines = [] - for content, metadata in result.items(): - line = f"• `{content}`" - - if comment := metadata.get("comment"): - line += f" - {comment}" - - lines.append(line) - lines = sorted(lines) - - # Build the embed - list_type_plural = list_type.lower().replace("_", " ").title() + "s" - embed = Embed( - title=f"{allow_type.title()}ed {list_type_plural} ({len(result)} total)", - colour=Colour.blue() - ) - log.trace(f"Trying to list {len(result)} items from the {list_type.lower()} {allow_type}") - - if result: - await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) - else: - embed.description = "Hmmm, seems like there's nothing here yet." - await ctx.send(embed=embed) - await ctx.message.add_reaction("❌") - - async def _sync_data(self, ctx: Context) -> None: - """Syncs the filterlists with the API.""" - try: - log.trace("Attempting to sync FilterList cache with data from the API.") - await self.bot.cache_filter_list_data() - await ctx.message.add_reaction("✅") - except ResponseCodeError as e: - log.debug( - f"{ctx.author} tried to sync FilterList cache data but " - f"the API raised an unexpected error: {e}" - ) - await ctx.message.add_reaction("❌") - - @staticmethod - async def _validate_guild_invite(ctx: Context, invite: str) -> dict: - """ - Validates a guild invite, and returns the guild info as a dict. - - Will raise a BadArgument if the guild invite is invalid. - """ - log.trace(f"Attempting to validate whether or not {invite} is a guild invite.") - validator = ValidDiscordServerInvite() - guild_data = await validator.convert(ctx, invite) - - # If we make it this far without raising a BadArgument, the invite is - # valid. Let's return a dict of guild information. - log.trace(f"{invite} validated as server invite. Converting to ID.") - return guild_data - - @group(aliases=("allowlist", "allow", "al", "wl")) - async def whitelist(self, ctx: Context) -> None: - """Group for whitelisting commands.""" - if not ctx.invoked_subcommand: - await ctx.send_help(ctx.command) - - @group(aliases=("denylist", "deny", "bl", "dl")) - async def blacklist(self, ctx: Context) -> None: - """Group for blacklisting commands.""" - if not ctx.invoked_subcommand: - await ctx.send_help(ctx.command) - - @whitelist.command(name="add", aliases=("a", "set")) - async def allow_add( - self, - ctx: Context, - list_type: ValidFilterListType, - content: str, - *, - comment: Optional[str] = None, - ) -> None: - """Add an item to the specified allowlist.""" - await self._add_data(ctx, True, list_type, content, comment) - - @blacklist.command(name="add", aliases=("a", "set")) - async def deny_add( - self, - ctx: Context, - list_type: ValidFilterListType, - content: str, - *, - comment: Optional[str] = None, - ) -> None: - """Add an item to the specified denylist.""" - await self._add_data(ctx, False, list_type, content, comment) - - @whitelist.command(name="remove", aliases=("delete", "rm",)) - async def allow_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: - """Remove an item from the specified allowlist.""" - await self._delete_data(ctx, True, list_type, content) - - @blacklist.command(name="remove", aliases=("delete", "rm",)) - async def deny_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: - """Remove an item from the specified denylist.""" - await self._delete_data(ctx, False, list_type, content) - - @whitelist.command(name="get", aliases=("list", "ls", "fetch", "show")) - async def allow_get(self, ctx: Context, list_type: ValidFilterListType) -> None: - """Get the contents of a specified allowlist.""" - await self._list_all_data(ctx, True, list_type) - - @blacklist.command(name="get", aliases=("list", "ls", "fetch", "show")) - async def deny_get(self, ctx: Context, list_type: ValidFilterListType) -> None: - """Get the contents of a specified denylist.""" - await self._list_all_data(ctx, False, list_type) - - @whitelist.command(name="sync", aliases=("s",)) - async def allow_sync(self, ctx: Context) -> None: - """Syncs both allowlists and denylists with the API.""" - await self._sync_data(ctx) - - @blacklist.command(name="sync", aliases=("s",)) - async def deny_sync(self, ctx: Context) -> None: - """Syncs both allowlists and denylists with the API.""" - await self._sync_data(ctx) - - async def cog_check(self, ctx: Context) -> bool: - """Only allow moderators to invoke the commands in this cog.""" - return await has_any_role(*constants.MODERATION_ROLES).predicate(ctx) - - -async def setup(bot: Bot) -> None: - """Load the FilterLists cog.""" - await bot.add_cog(FilterLists(bot)) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py deleted file mode 100644 index ca6ad00644..0000000000 --- a/bot/exts/filters/filtering.py +++ /dev/null @@ -1,735 +0,0 @@ -import asyncio -import re -import unicodedata -import urllib.parse -from datetime import timedelta -from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Tuple, Union - -import arrow -import dateutil.parser -import regex -import tldextract -from async_rediscache import RedisCache -from botcore.site_api import ResponseCodeError -from botcore.utils import scheduling -from botcore.utils.regex import DISCORD_INVITE -from dateutil.relativedelta import relativedelta -from discord import ChannelType, Colour, Embed, Forbidden, HTTPException, Member, Message, NotFound, TextChannel -from discord.ext.commands import Cog -from discord.utils import escape_markdown - -from bot.bot import Bot -from bot.constants import Bot as BotConfig, Channels, Colours, Filter, Guild, Icons, URLs -from bot.exts.events.code_jams._channels import CATEGORY_NAME as JAM_CATEGORY_NAME -from bot.exts.moderation.modlog import ModLog -from bot.log import get_logger -from bot.utils.messages import format_user - -log = get_logger(__name__) - -# Regular expressions -CODE_BLOCK_RE = re.compile( - r"(?P``?)[^`]+?(?P=delim)(?!`+)" # Inline codeblock - r"|```(.+?)```", # Multiline codeblock - re.DOTALL | re.MULTILINE -) -EVERYONE_PING_RE = re.compile(rf"@everyone|<@&{Guild.id}>|@here") -SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL) -URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) - -# Exclude variation selectors from zalgo because they're actually invisible. -VARIATION_SELECTORS = r"\uFE00-\uFE0F\U000E0100-\U000E01EF" -INVISIBLE_RE = regex.compile(rf"[{VARIATION_SELECTORS}\p{{UNASSIGNED}}\p{{FORMAT}}\p{{CONTROL}}--\s]", regex.V1) -ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIATION_SELECTORS}]]", regex.V1) - -# Other constants. -DAYS_BETWEEN_ALERTS = 3 -OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) - -# Autoban -LINK_PASSWORD = "https://support.discord.com/hc/en-us/articles/218410947-I-forgot-my-Password-Where-can-I-set-a-new-one" -LINK_2FA = "https://support.discord.com/hc/en-us/articles/219576828-Setting-up-Two-Factor-Authentication" -AUTO_BAN_REASON = ( - "Your account has been used to send links to a phishing website. You have been automatically banned. " - "If you are not aware of sending them, that means your account has been compromised.\n\n" - - f"Here is a guide from Discord on [how to change your password]({LINK_PASSWORD}).\n\n" - - f"We also highly recommend that you [enable 2 factor authentication on your account]({LINK_2FA}), " - "for heightened security.\n\n" - - "Once you have changed your password, feel free to follow the instructions at the bottom of " - "this message to appeal your ban." -) -AUTO_BAN_DURATION = timedelta(days=4) - -FilterMatch = Union[re.Match, dict, bool, List[Embed]] - - -class Stats(NamedTuple): - """Additional stats on a triggered filter to append to a mod log.""" - - message_content: str - additional_embeds: Optional[List[Embed]] - - -class Filtering(Cog): - """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" - - # Redis cache mapping a user ID to the last timestamp a bad nickname alert was sent - name_alerts = RedisCache() - - def __init__(self, bot: Bot): - self.bot = bot - self.scheduler = scheduling.Scheduler(self.__class__.__name__) - self.name_lock = asyncio.Lock() - - staff_mistake_str = "If you believe this was a mistake, please let staff know!" - self.filters = { - "filter_zalgo": { - "enabled": Filter.filter_zalgo, - "function": self._has_zalgo, - "type": "filter", - "content_only": True, - "user_notification": Filter.notify_user_zalgo, - "notification_msg": ( - "Your post has been removed for abusing Unicode character rendering (aka Zalgo text). " - f"{staff_mistake_str}" - ), - "schedule_deletion": False - }, - "filter_invites": { - "enabled": Filter.filter_invites, - "function": self._has_invites, - "type": "filter", - "content_only": True, - "user_notification": Filter.notify_user_invites, - "notification_msg": ( - f"Per Rule 6, your invite link has been removed. {staff_mistake_str}\n\n" - r"Our server rules can be found here: " - ), - "schedule_deletion": False - }, - "filter_domains": { - "enabled": Filter.filter_domains, - "function": self._has_urls, - "type": "filter", - "content_only": True, - "user_notification": Filter.notify_user_domains, - "notification_msg": ( - f"Your URL has been removed because it matched a blacklisted domain. {staff_mistake_str}" - ), - "schedule_deletion": False - }, - "watch_regex": { - "enabled": Filter.watch_regex, - "function": self._has_watch_regex_match, - "type": "watchlist", - "content_only": True, - "schedule_deletion": True - }, - "watch_rich_embeds": { - "enabled": Filter.watch_rich_embeds, - "function": self._has_rich_embed, - "type": "watchlist", - "content_only": False, - "schedule_deletion": False - }, - "filter_everyone_ping": { - "enabled": Filter.filter_everyone_ping, - "function": self._has_everyone_ping, - "type": "filter", - "content_only": True, - "user_notification": Filter.notify_user_everyone_ping, - "notification_msg": ( - "Please don't try to ping `@everyone` or `@here`. " - f"Your message has been removed. {staff_mistake_str}" - ), - "schedule_deletion": False, - "ping_everyone": False - }, - } - - async def cog_unload(self) -> None: - """Cancel scheduled tasks.""" - self.scheduler.cancel_all() - - def _get_filterlist_items(self, list_type: str, *, allowed: bool) -> list: - """Fetch items from the filter_list_cache.""" - return self.bot.filter_list_cache[f"{list_type.upper()}.{allowed}"].keys() - - def _get_filterlist_value(self, list_type: str, value: Any, *, allowed: bool) -> dict: - """Fetch one specific value from filter_list_cache.""" - return self.bot.filter_list_cache[f"{list_type.upper()}.{allowed}"][value] - - @staticmethod - def _expand_spoilers(text: str) -> str: - """Return a string containing all interpretations of a spoilered message.""" - split_text = SPOILER_RE.split(text) - return ''.join( - split_text[0::2] + split_text[1::2] + split_text - ) - - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") - - @Cog.listener() - async def on_message(self, msg: Message) -> None: - """Invoke message filter for new messages.""" - await self._filter_message(msg) - - # Ignore webhook messages. - if msg.webhook_id is None: - await self.check_bad_words_in_name(msg.author) - - @Cog.listener() - async def on_message_edit(self, before: Message, after: Message) -> None: - """ - Invoke message filter for message edits. - - Also calculates the time delta from the previous edit or when message was sent if there's no prior edits. - """ - # We only care about changes to the message contents/attachments and embed additions, not pin status etc. - if all(( - before.content == after.content, # content hasn't changed - before.attachments == after.attachments, # attachments haven't changed - len(before.embeds) >= len(after.embeds) # embeds haven't been added - )): - return - - if not before.edited_at: - delta = relativedelta(after.edited_at, before.created_at).microseconds - else: - delta = relativedelta(after.edited_at, before.edited_at).microseconds - await self._filter_message(after, delta) - - @Cog.listener() - async def on_voice_state_update(self, member: Member, *_) -> None: - """Checks for bad words in usernames when users join, switch or leave a voice channel.""" - await self.check_bad_words_in_name(member) - - def get_name_match(self, name: str) -> Optional[re.Match]: - """Check bad words from passed string (name). Return the first match found.""" - normalised_name = unicodedata.normalize("NFKC", name) - cleaned_normalised_name = "".join([c for c in normalised_name if not unicodedata.combining(c)]) - - # Run filters against normalised, cleaned normalised and the original name, - # in case we have filters for one but not the other. - names_to_check = (name, normalised_name, cleaned_normalised_name) - - watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) - for pattern in watchlist_patterns: - for name in names_to_check: - if match := re.search(pattern, name, flags=re.IGNORECASE): - return match - return None - - async def check_send_alert(self, member: Member) -> bool: - """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" - if last_alert := await self.name_alerts.get(member.id): - last_alert = arrow.get(last_alert) - if arrow.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: - log.trace(f"Last alert was too recent for {member}'s nickname.") - return False - - return True - - async def check_bad_words_in_name(self, member: Member) -> None: - """Send a mod alert every 3 days if a username still matches a watchlist pattern.""" - # Use lock to avoid race conditions - async with self.name_lock: - # Check if we recently alerted about this user first, - # to avoid running all the filter tokens against their name again. - if not await self.check_send_alert(member): - return - - # Check whether the users display name contains any words in our blacklist - match = self.get_name_match(member.display_name) - if not match: - return - - log.info(f"Sending bad nickname alert for '{member.display_name}' ({member.id}).") - - log_string = ( - f"**User:** {format_user(member)}\n" - f"**Display Name:** {escape_markdown(member.display_name)}\n" - f"**Bad Match:** {match.group()}" - ) - - await self.mod_log.send_log_message( - content=str(member.id), # quality-of-life improvement for mobile moderators - icon_url=Icons.token_removed, - colour=Colours.soft_red, - title="Username filtering alert", - text=log_string, - channel_id=Channels.mod_alerts, - thumbnail=member.display_avatar.url, - ping_everyone=True - ) - - # Update time when alert sent - await self.name_alerts.set(member.id, arrow.utcnow().timestamp()) - - async def filter_snekbox_output(self, result: str, msg: Message) -> bool: - """ - Filter the result of a snekbox command to see if it violates any of our rules, and then respond accordingly. - - Also requires the original message, to check whether to filter and for mod logs. - Returns whether a filter was triggered or not. - """ - filter_triggered = False - # Should we filter this message? - if self._check_filter(msg): - for filter_name, _filter in self.filters.items(): - # Is this specific filter enabled in the config? - # We also do not need to worry about filters that take the full message, - # since all we have is an arbitrary string. - if _filter["enabled"] and _filter["content_only"]: - filter_result = await _filter["function"](result) - reason = None - - if isinstance(filter_result, tuple): - match, reason = filter_result - else: - match = filter_result - - if match: - # If this is a filter (not a watchlist), we set the variable so we know - # that it has been triggered - if _filter["type"] == "filter": - filter_triggered = True - - stats = self._add_stats(filter_name, match, result) - await self._send_log(filter_name, _filter, msg, stats, reason, is_eval=True) - - break # We don't want multiple filters to trigger - - return filter_triggered - - async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: - """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" - # Should we filter this message? - if self._check_filter(msg): - for filter_name, _filter in self.filters.items(): - # Is this specific filter enabled in the config? - if _filter["enabled"]: - # Double trigger check for the embeds filter - if filter_name == "watch_rich_embeds": - # If the edit delta is less than 0.001 seconds, then we're probably dealing - # with a double filter trigger. - if delta is not None and delta < 100: - continue - - if filter_name in ("filter_invites", "filter_everyone_ping"): - # Disable invites filter in codejam team channels - category = getattr(msg.channel, "category", None) - if category and category.name == JAM_CATEGORY_NAME: - continue - - # Does the filter only need the message content or the full message? - if _filter["content_only"]: - payload = msg.content - else: - payload = msg - - result = await _filter["function"](payload) - reason = None - - if isinstance(result, tuple): - match, reason = result - else: - match = result - - if match: - is_private = msg.channel.type is ChannelType.private - - # If this is a filter (not a watchlist) and not in a DM, delete the message. - if _filter["type"] == "filter" and not is_private: - try: - # Embeds (can?) trigger both the `on_message` and `on_message_edit` - # event handlers, triggering filtering twice for the same message. - # - # If `on_message`-triggered filtering already deleted the message - # then `on_message_edit`-triggered filtering will raise exception - # since the message no longer exists. - # - # In addition, to avoid sending two notifications to the user, the - # logs, and mod_alert, we return if the message no longer exists. - await msg.delete() - except NotFound: - return - - # Notify the user if the filter specifies - if _filter["user_notification"]: - await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) - - # If the message is classed as offensive, we store it in the site db and - # it will be deleted after one week. - if _filter["schedule_deletion"] and not is_private: - delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() - data = { - 'id': msg.id, - 'channel_id': msg.channel.id, - 'delete_date': delete_date - } - - try: - await self.bot.api_client.post('bot/offensive-messages', json=data) - except ResponseCodeError as e: - if e.status == 400 and "already exists" in e.response_json.get("id", [""])[0]: - log.debug(f"Offensive message {msg.id} already exists.") - else: - log.error(f"Offensive message {msg.id} failed to post: {e}") - else: - self.schedule_msg_delete(data) - log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") - - stats = self._add_stats(filter_name, match, msg.content) - - # If the filter reason contains `[autoban]`, we want to auto-ban the user. - # Also pass this to _send_log so mods are not pinged filter matches that are auto-actioned - autoban = reason and "[autoban]" in reason.lower() - if not autoban and filter_name == "filter_invites" and isinstance(result, dict): - autoban = any( - "[autoban]" in invite_info["reason"].lower() - for invite_info in result.values() - if invite_info.get("reason") - ) - - await self._send_log(filter_name, _filter, msg, stats, reason, autoban=autoban) - - if autoban: - # Create a new context, with the author as is the bot, and the channel as #mod-alerts. - # This sends the ban confirmation directly under watchlist trigger embed, to inform - # mods that the user was auto-banned for the message. - context = await self.bot.get_context(msg) - context.guild = self.bot.get_guild(Guild.id) - context.author = context.guild.get_member(self.bot.user.id) - context.channel = self.bot.get_channel(Channels.mod_alerts) - context.command = self.bot.get_command("tempban") - - await context.invoke( - context.command, - msg.author, - arrow.utcnow() + AUTO_BAN_DURATION, - reason=AUTO_BAN_REASON - ) - - break # We don't want multiple filters to trigger - - async def _send_log( - self, - filter_name: str, - _filter: Dict[str, Any], - msg: Message, - stats: Stats, - reason: Optional[str] = None, - *, - is_eval: bool = False, - autoban: bool = False, - ) -> None: - """Send a mod log for a triggered filter.""" - if msg.channel.type is ChannelType.private: - channel_str = "via DM" - ping_everyone = False - else: - channel_str = f"in {msg.channel.mention}" - # Allow specific filters to override ping_everyone - ping_everyone = Filter.ping_everyone and _filter.get("ping_everyone", True) - - content = str(msg.author.id) # quality-of-life improvement for mobile moderators - - # If we are going to autoban, we don't want to ping and don't need the user ID - if autoban: - ping_everyone = False - content = None - - eval_msg = f"using {BotConfig.prefix}eval " if is_eval else "" - footer = f"Reason: {reason}" if reason else None - message = ( - f"The {filter_name} {_filter['type']} was triggered by {format_user(msg.author)} " - f"{channel_str} {eval_msg}with [the following message]({msg.jump_url}):\n\n" - f"{stats.message_content}" - ) - - log.debug(message) - - # Send pretty mod log embed to mod-alerts - await self.mod_log.send_log_message( - content=content, - icon_url=Icons.filtering, - colour=Colour(Colours.soft_red), - title=f"{_filter['type'].title()} triggered!", - text=message, - thumbnail=msg.author.display_avatar.url, - channel_id=Channels.mod_alerts, - ping_everyone=ping_everyone, - additional_embeds=stats.additional_embeds, - footer=footer, - ) - - def _add_stats(self, name: str, match: FilterMatch, content: str) -> Stats: - """Adds relevant statistical information to the relevant filter and increments the bot's stats.""" - # Word and match stats for watch_regex - if name == "watch_regex": - surroundings = match.string[max(match.start() - 10, 0): match.end() + 10] - message_content = ( - f"**Match:** '{match[0]}'\n" - f"**Location:** '...{escape_markdown(surroundings)}...'\n" - f"\n**Original Message:**\n{escape_markdown(content)}" - ) - else: # Use original content - message_content = content - - additional_embeds = None - - self.bot.stats.incr(f"filters.{name}") - - # The function returns True for invalid invites. - # They have no data so additional embeds can't be created for them. - if name == "filter_invites" and match is not True: - additional_embeds = [] - for _, data in match.items(): - reason = f"Reason: {data['reason']} | " if data.get('reason') else "" - embed = Embed(description=( - f"**Members:**\n{data['members']}\n" - f"**Active:**\n{data['active']}" - )) - embed.set_author(name=data["name"]) - embed.set_thumbnail(url=data["icon"]) - embed.set_footer(text=f"{reason}Guild ID: {data['id']}") - additional_embeds.append(embed) - - elif name == "watch_rich_embeds": - additional_embeds = match - - return Stats(message_content, additional_embeds) - - @staticmethod - def _check_filter(msg: Message) -> bool: - """Check whitelists to see if we should filter this message.""" - role_whitelisted = False - - if type(msg.author) is Member: # Only Member has roles, not User. - for role in msg.author.roles: - if role.id in Filter.role_whitelist: - role_whitelisted = True - - return ( - msg.channel.id not in Filter.channel_whitelist # Channel not in whitelist - and not role_whitelisted # Role not in whitelist - and not msg.author.bot # Author not a bot - ) - - async def _has_watch_regex_match(self, text: str) -> Tuple[Union[bool, re.Match], Optional[str]]: - """ - Return True if `text` matches any regex from `word_watchlist` or `token_watchlist` configs. - - `word_watchlist`'s patterns are placed between word boundaries while `token_watchlist` is - matched as-is. Spoilers are expanded, if any, and URLs are ignored. - Second return value is a reason written to database about blacklist entry (can be None). - """ - if SPOILER_RE.search(text): - text = self._expand_spoilers(text) - - text = self.clean_input(text) - - watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) - for pattern in watchlist_patterns: - match = re.search(pattern, text, flags=re.IGNORECASE) - if match: - return match, self._get_filterlist_value('filter_token', pattern, allowed=False)['comment'] - - return False, None - - async def _has_urls(self, text: str) -> Tuple[bool, Optional[str]]: - """ - Returns True if the text contains one of the blacklisted URLs from the config file. - - Second return value is a reason of URL blacklisting (can be None). - """ - text = self.clean_input(text) - - domain_blacklist = self._get_filterlist_items("domain_name", allowed=False) - for match in URL_RE.finditer(text): - for url in domain_blacklist: - if url.lower() in match.group(1).lower(): - blacklisted_parsed = tldextract.extract(url.lower()) - url_parsed = tldextract.extract(match.group(1).lower()) - if blacklisted_parsed.registered_domain == url_parsed.registered_domain: - return True, self._get_filterlist_value("domain_name", url, allowed=False)["comment"] - return False, None - - @staticmethod - async def _has_zalgo(text: str) -> bool: - """ - Returns True if the text contains zalgo characters. - - Zalgo range is \u0300 – \u036F and \u0489. - """ - return bool(ZALGO_RE.search(text)) - - async def _has_invites(self, text: str) -> Union[dict, bool]: - """ - Checks if there's any invites in the text content that aren't in the guild whitelist. - - If any are detected, a dictionary of invite data is returned, with a key per invite. - If none are detected, False is returned. - If we are unable to process an invite, True is returned. - - Attempts to catch some of common ways to try to cheat the system. - """ - text = self.clean_input(text) - - # Remove backslashes to prevent escape character aroundfuckery like - # discord\.gg/gdudes-pony-farm - text = text.replace("\\", "") - - invites = [m.group("invite") for m in DISCORD_INVITE.finditer(text)] - invite_data = dict() - for invite in invites: - invite = urllib.parse.quote_plus(invite.rstrip("/")) - if invite in invite_data: - continue - - response = await self.bot.http_session.get( - f"{URLs.discord_invite_api}/{invite}", params={"with_counts": "true"} - ) - response = await response.json() - guild = response.get("guild") - if guild is None: - # Lack of a "guild" key in the JSON response indicates either an group DM invite, an - # expired invite, or an invalid invite. The API does not currently differentiate - # between invalid and expired invites - return True - - guild_id = guild.get("id") - guild_invite_whitelist = self._get_filterlist_items("guild_invite", allowed=True) - guild_invite_blacklist = self._get_filterlist_items("guild_invite", allowed=False) - - # Is this invite allowed? - guild_partnered_or_verified = ( - 'PARTNERED' in guild.get("features", []) - or 'VERIFIED' in guild.get("features", []) - ) - invite_not_allowed = ( - guild_id in guild_invite_blacklist # Blacklisted guilds are never permitted. - or guild_id not in guild_invite_whitelist # Whitelisted guilds are always permitted. - and not guild_partnered_or_verified # Otherwise guilds have to be Verified or Partnered. - ) - - if invite_not_allowed: - reason = None - if guild_id in guild_invite_blacklist: - reason = self._get_filterlist_value("guild_invite", guild_id, allowed=False)["comment"] - - guild_icon_hash = guild["icon"] - guild_icon = ( - "https://cdn.discordapp.com/icons/" - f"{guild_id}/{guild_icon_hash}.png?size=512" - ) - - invite_data[invite] = { - "name": guild["name"], - "id": guild['id'], - "icon": guild_icon, - "members": response["approximate_member_count"], - "active": response["approximate_presence_count"], - "reason": reason - } - - return invite_data if invite_data else False - - @staticmethod - async def _has_rich_embed(msg: Message) -> Union[bool, List[Embed]]: - """Determines if `msg` contains any rich embeds not auto-generated from a URL.""" - if msg.embeds: - for embed in msg.embeds: - if embed.type == "rich": - urls = URL_RE.findall(msg.content) - if not embed.url or embed.url not in urls: - # If `embed.url` does not exist or if `embed.url` is not part of the content - # of the message, it's unlikely to be an auto-generated embed by Discord. - return msg.embeds - else: - log.trace( - "Found a rich embed sent by a regular user account, " - "but it was likely just an automatic URL embed." - ) - return False - return False - - @staticmethod - async def _has_everyone_ping(text: str) -> bool: - """Determines if `msg` contains an @everyone or @here ping outside of a codeblock.""" - # First pass to avoid running re.sub on every message - if not EVERYONE_PING_RE.search(text): - return False - - content_without_codeblocks = CODE_BLOCK_RE.sub("", text) - return bool(EVERYONE_PING_RE.search(content_without_codeblocks)) - - async def notify_member(self, filtered_member: Member, reason: str, channel: TextChannel) -> None: - """ - Notify filtered_member about a moderation action with the reason str. - - First attempts to DM the user, fall back to in-channel notification if user has DMs disabled - """ - try: - await filtered_member.send(reason) - except Forbidden: - await channel.send(f"{filtered_member.mention} {reason}") - - def schedule_msg_delete(self, msg: dict) -> None: - """Delete an offensive message once its deletion date is reached.""" - delete_at = dateutil.parser.isoparse(msg['delete_date']) - self.scheduler.schedule_at(delete_at, msg['id'], self.delete_offensive_msg(msg)) - - async def cog_load(self) -> None: - """Get all the pending message deletion from the API and reschedule them.""" - await self.bot.wait_until_ready() - response = await self.bot.api_client.get('bot/offensive-messages',) - - now = arrow.utcnow() - - for msg in response: - delete_at = dateutil.parser.isoparse(msg['delete_date']) - - if delete_at < now: - await self.delete_offensive_msg(msg) - else: - self.schedule_msg_delete(msg) - - async def delete_offensive_msg(self, msg: Mapping[str, int]) -> None: - """Delete an offensive message, and then delete it from the db.""" - try: - channel = self.bot.get_channel(msg['channel_id']) - if channel: - msg_obj = await channel.fetch_message(msg['id']) - await msg_obj.delete() - except NotFound: - log.info( - f"Tried to delete message {msg['id']}, but the message can't be found " - f"(it has been probably already deleted)." - ) - except HTTPException as e: - log.warning(f"Failed to delete message {msg['id']}: status {e.status}") - - await self.bot.api_client.delete(f'bot/offensive-messages/{msg["id"]}') - log.info(f"Deleted the offensive message with id {msg['id']}.") - - @staticmethod - def clean_input(string: str) -> str: - """Remove zalgo and invisible characters from `string`.""" - # For future consideration: remove characters in the Mc, Sk, and Lm categories too. - # Can be normalised with form C to merge char + combining char into a single char to avoid - # removing legit diacritics, but this would open up a way to bypass filters. - no_zalgo = ZALGO_RE.sub("", string) - return INVISIBLE_RE.sub("", no_zalgo) - - -async def setup(bot: Bot) -> None: - """Load the Filtering cog.""" - await bot.add_cog(Filtering(bot)) diff --git a/bot/exts/filters/security.py b/bot/exts/filters/security.py deleted file mode 100644 index 27e4d97525..0000000000 --- a/bot/exts/filters/security.py +++ /dev/null @@ -1,30 +0,0 @@ -from discord.ext.commands import Cog, Context, NoPrivateMessage - -from bot.bot import Bot -from bot.log import get_logger - -log = get_logger(__name__) - - -class Security(Cog): - """Security-related helpers.""" - - def __init__(self, bot: Bot): - self.bot = bot - self.bot.check(self.check_not_bot) # Global commands check - no bots can run any commands at all - self.bot.check(self.check_on_guild) # Global commands check - commands can't be run in a DM - - def check_not_bot(self, ctx: Context) -> bool: - """Check if the context is a bot user.""" - return not ctx.author.bot - - def check_on_guild(self, ctx: Context) -> bool: - """Check if the context is in a guild.""" - if ctx.guild is None: - raise NoPrivateMessage("This command cannot be used in private messages.") - return True - - -async def setup(bot: Bot) -> None: - """Load the Security cog.""" - await bot.add_cog(Security(bot)) diff --git a/bot/exts/filters/token_remover.py b/bot/exts/filters/token_remover.py deleted file mode 100644 index a0d5aa7b60..0000000000 --- a/bot/exts/filters/token_remover.py +++ /dev/null @@ -1,233 +0,0 @@ -import base64 -import re -import typing as t - -from discord import Colour, Message, NotFound -from discord.ext.commands import Cog - -from bot import utils -from bot.bot import Bot -from bot.constants import Channels, Colours, Event, Icons -from bot.exts.moderation.modlog import ModLog -from bot.log import get_logger -from bot.utils.members import get_or_fetch_member -from bot.utils.messages import format_user - -log = get_logger(__name__) - -LOG_MESSAGE = ( - "Censored a seemingly valid token sent by {author} in {channel}, " - "token was `{user_id}.{timestamp}.{hmac}`" -) -UNKNOWN_USER_LOG_MESSAGE = "Decoded user ID: `{user_id}` (Not present in server)." -KNOWN_USER_LOG_MESSAGE = ( - "Decoded user ID: `{user_id}` **(Present in server)**.\n" - "This matches `{user_name}` and means this is likely a valid **{kind}** token." -) -DELETION_MESSAGE_TEMPLATE = ( - "Hey {mention}! I noticed you posted a seemingly valid Discord API " - "token in your message and have removed your message. " - "This means that your token has been **compromised**. " - "Please change your token **immediately** at: " - "\n\n" - "Feel free to re-post it with the token removed. " - "If you believe this was a mistake, please let us know!" -) -DISCORD_EPOCH = 1_420_070_400 -TOKEN_EPOCH = 1_293_840_000 - -# Three parts delimited by dots: user ID, creation timestamp, HMAC. -# The HMAC isn't parsed further, but it's in the regex to ensure it at least exists in the string. -# Each part only matches base64 URL-safe characters. -# Padding has never been observed, but the padding character '=' is matched just in case. -TOKEN_RE = re.compile(r"([\w\-=]+)\.([\w\-=]+)\.([\w\-=]+)", re.ASCII) - - -class Token(t.NamedTuple): - """A Discord Bot token.""" - - user_id: str - timestamp: str - hmac: str - - -class TokenRemover(Cog): - """Scans messages for potential discord.py bot tokens and removes them.""" - - def __init__(self, bot: Bot): - self.bot = bot - - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") - - @Cog.listener() - async def on_message(self, msg: Message) -> None: - """ - Check each message for a string that matches Discord's token pattern. - - See: https://discordapp.com/developers/docs/reference#snowflakes - """ - # Ignore DMs; can't delete messages in there anyway. - if not msg.guild or msg.author.bot: - return - - found_token = self.find_token_in_message(msg) - if found_token: - await self.take_action(msg, found_token) - - @Cog.listener() - async def on_message_edit(self, before: Message, after: Message) -> None: - """ - Check each edit for a string that matches Discord's token pattern. - - See: https://discordapp.com/developers/docs/reference#snowflakes - """ - await self.on_message(after) - - async def take_action(self, msg: Message, found_token: Token) -> None: - """Remove the `msg` containing the `found_token` and send a mod log message.""" - self.mod_log.ignore(Event.message_delete, msg.id) - - try: - await msg.delete() - except NotFound: - log.debug(f"Failed to remove token in message {msg.id}: message already deleted.") - return - - await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) - - log_message = self.format_log_message(msg, found_token) - userid_message, mention_everyone = await self.format_userid_log_message(msg, found_token) - log.debug(log_message) - - # Send pretty mod log embed to mod-alerts - await self.mod_log.send_log_message( - icon_url=Icons.token_removed, - colour=Colour(Colours.soft_red), - title="Token removed!", - text=log_message + "\n" + userid_message, - thumbnail=msg.author.display_avatar.url, - channel_id=Channels.mod_alerts, - ping_everyone=mention_everyone, - ) - - self.bot.stats.incr("tokens.removed_tokens") - - @classmethod - async def format_userid_log_message(cls, msg: Message, token: Token) -> t.Tuple[str, bool]: - """ - Format the portion of the log message that includes details about the detected user ID. - - If the user is resolved to a member, the format includes the user ID, name, and the - kind of user detected. - - If we resolve to a member and it is not a bot, we also return True to ping everyone. - - Returns a tuple of (log_message, mention_everyone) - """ - user_id = cls.extract_user_id(token.user_id) - user = await get_or_fetch_member(msg.guild, user_id) - - if user: - return KNOWN_USER_LOG_MESSAGE.format( - user_id=user_id, - user_name=str(user), - kind="BOT" if user.bot else "USER", - ), True - else: - return UNKNOWN_USER_LOG_MESSAGE.format(user_id=user_id), False - - @staticmethod - def format_log_message(msg: Message, token: Token) -> str: - """Return the generic portion of the log message to send for `token` being censored in `msg`.""" - return LOG_MESSAGE.format( - author=format_user(msg.author), - channel=msg.channel.mention, - user_id=token.user_id, - timestamp=token.timestamp, - hmac='x' * (len(token.hmac) - 3) + token.hmac[-3:], - ) - - @classmethod - def find_token_in_message(cls, msg: Message) -> t.Optional[Token]: - """Return a seemingly valid token found in `msg` or `None` if no token is found.""" - # Use finditer rather than search to guard against method calls prematurely returning the - # token check (e.g. `message.channel.send` also matches our token pattern) - for match in TOKEN_RE.finditer(msg.content): - token = Token(*match.groups()) - if ( - (cls.extract_user_id(token.user_id) is not None) - and cls.is_valid_timestamp(token.timestamp) - and cls.is_maybe_valid_hmac(token.hmac) - ): - # Short-circuit on first match - return token - - # No matching substring - return - - @staticmethod - def extract_user_id(b64_content: str) -> t.Optional[int]: - """Return a user ID integer from part of a potential token, or None if it couldn't be decoded.""" - b64_content = utils.pad_base64(b64_content) - - try: - decoded_bytes = base64.urlsafe_b64decode(b64_content) - string = decoded_bytes.decode('utf-8') - if not (string.isascii() and string.isdigit()): - # This case triggers if there are fancy unicode digits in the base64 encoding, - # that means it's not a valid user id. - return None - return int(string) - except ValueError: - return None - - @staticmethod - def is_valid_timestamp(b64_content: str) -> bool: - """ - Return True if `b64_content` decodes to a valid timestamp. - - If the timestamp is greater than the Discord epoch, it's probably valid. - See: https://i.imgur.com/7WdehGn.png - """ - b64_content = utils.pad_base64(b64_content) - - try: - decoded_bytes = base64.urlsafe_b64decode(b64_content) - timestamp = int.from_bytes(decoded_bytes, byteorder="big") - except ValueError as e: - log.debug(f"Failed to decode token timestamp '{b64_content}': {e}") - return False - - # Seems like newer tokens don't need the epoch added, but add anyway since an upper bound - # is not checked. - if timestamp + TOKEN_EPOCH >= DISCORD_EPOCH: - return True - else: - log.debug(f"Invalid token timestamp '{b64_content}': smaller than Discord epoch") - return False - - @staticmethod - def is_maybe_valid_hmac(b64_content: str) -> bool: - """ - Determine if a given HMAC portion of a token is potentially valid. - - If the HMAC has 3 or less characters, it's probably a dummy value like "xxxxxxxxxx", - and thus the token can probably be skipped. - """ - unique = len(set(b64_content.lower())) - if unique <= 3: - log.debug( - f"Considering the HMAC {b64_content} a dummy because it has {unique}" - " case-insensitively unique characters" - ) - return False - else: - return True - - -async def setup(bot: Bot) -> None: - """Load the TokenRemover cog.""" - await bot.add_cog(TokenRemover(bot)) diff --git a/bot/exts/filters/webhook_remover.py b/bot/exts/filters/webhook_remover.py deleted file mode 100644 index b426138048..0000000000 --- a/bot/exts/filters/webhook_remover.py +++ /dev/null @@ -1,94 +0,0 @@ -import re - -from discord import Colour, Message, NotFound -from discord.ext.commands import Cog - -from bot.bot import Bot -from bot.constants import Channels, Colours, Event, Icons -from bot.exts.moderation.modlog import ModLog -from bot.log import get_logger -from bot.utils.messages import format_user - -WEBHOOK_URL_RE = re.compile( - r"((?:https?:\/\/)?(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/)\S+\/?", - re.IGNORECASE -) - -ALERT_MESSAGE_TEMPLATE = ( - "{user}, looks like you posted a Discord webhook URL. Therefore, your " - "message has been removed, and your webhook has been deleted. " - "You can re-create it if you wish to. If you believe this was a " - "mistake, please let us know." -) - -log = get_logger(__name__) - - -class WebhookRemover(Cog): - """Scan messages to detect Discord webhooks links.""" - - def __init__(self, bot: Bot): - self.bot = bot - - @property - def mod_log(self) -> ModLog: - """Get current instance of `ModLog`.""" - return self.bot.get_cog("ModLog") - - async def delete_and_respond(self, msg: Message, redacted_url: str, *, webhook_deleted: bool) -> None: - """Delete `msg` and send a warning that it contained the Discord webhook `redacted_url`.""" - # Don't log this, due internal delete, not by user. Will make different entry. - self.mod_log.ignore(Event.message_delete, msg.id) - - try: - await msg.delete() - except NotFound: - log.debug(f"Failed to remove webhook in message {msg.id}: message already deleted.") - return - - await msg.channel.send(ALERT_MESSAGE_TEMPLATE.format(user=msg.author.mention)) - if webhook_deleted: - delete_state = "The webhook was successfully deleted." - else: - delete_state = "There was an error when deleting the webhook, it might have already been removed." - message = ( - f"{format_user(msg.author)} posted a Discord webhook URL to {msg.channel.mention}. {delete_state} " - f"Webhook URL was `{redacted_url}`" - ) - log.debug(message) - - # Send entry to moderation alerts. - await self.mod_log.send_log_message( - icon_url=Icons.token_removed, - colour=Colour(Colours.soft_red), - title="Discord webhook URL removed!", - text=message, - thumbnail=msg.author.display_avatar.url, - channel_id=Channels.mod_alerts - ) - - self.bot.stats.incr("tokens.removed_webhooks") - - @Cog.listener() - async def on_message(self, msg: Message) -> None: - """Check if a Discord webhook URL is in `message`.""" - # Ignore DMs; can't delete messages in there anyway. - if not msg.guild or msg.author.bot: - return - - matches = WEBHOOK_URL_RE.search(msg.content) - if matches: - async with self.bot.http_session.delete(matches[0]) as resp: - # The Discord API Returns a 204 NO CONTENT response on success. - deleted_successfully = resp.status == 204 - await self.delete_and_respond(msg, matches[1] + "xxx", webhook_deleted=deleted_successfully) - - @Cog.listener() - async def on_message_edit(self, before: Message, after: Message) -> None: - """Check if a Discord webhook URL is in the edited message `after`.""" - await self.on_message(after) - - -async def setup(bot: Bot) -> None: - """Load `WebhookRemover` cog.""" - await bot.add_cog(WebhookRemover(bot)) diff --git a/bot/exts/info/codeblock/_cog.py b/bot/exts/info/codeblock/_cog.py index 9027105d9d..cc58621315 100644 --- a/bot/exts/info/codeblock/_cog.py +++ b/bot/exts/info/codeblock/_cog.py @@ -8,8 +8,6 @@ from bot import constants from bot.bot import Bot -from bot.exts.filters.token_remover import TokenRemover -from bot.exts.filters.webhook_remover import WEBHOOK_URL_RE from bot.exts.info.codeblock._instructions import get_instructions from bot.log import get_logger from bot.utils import has_lines @@ -135,8 +133,6 @@ def should_parse(self, message: discord.Message) -> bool: not message.author.bot and self.is_valid_channel(message.channel) and has_lines(message.content, constants.CodeBlock.minimum_lines) - and not TokenRemover.find_token_in_message(message) - and not WEBHOOK_URL_RE.search(message.content) ) @Cog.listener() diff --git a/bot/exts/moderation/watchchannels/_watchchannel.py b/bot/exts/moderation/watchchannels/_watchchannel.py index 46f9c296e9..30b1f342b3 100644 --- a/bot/exts/moderation/watchchannels/_watchchannel.py +++ b/bot/exts/moderation/watchchannels/_watchchannel.py @@ -14,8 +14,6 @@ from bot.bot import Bot from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons -from bot.exts.filters.token_remover import TokenRemover -from bot.exts.filters.webhook_remover import WEBHOOK_URL_RE from bot.exts.moderation.modlog import ModLog from bot.log import CustomLogger, get_logger from bot.pagination import LinePaginator @@ -235,9 +233,7 @@ async def relay_message(self, msg: Message) -> None: await self.send_header(msg) - if TokenRemover.find_token_in_message(msg) or WEBHOOK_URL_RE.search(msg.content): - cleaned_content = "Content is censored because it contains a bot or webhook token." - elif cleaned_content := msg.clean_content: + if cleaned_content := msg.clean_content: # Put all non-media URLs in a code block to prevent embeds media_urls = {embed.url for embed in msg.embeds if embed.type in ("image", "video")} for url in URL_RE.findall(cleaned_content): diff --git a/bot/rules/__init__.py b/bot/rules/__init__.py deleted file mode 100644 index a01ceae73d..0000000000 --- a/bot/rules/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# flake8: noqa - -from .attachments import apply as apply_attachments -from .burst import apply as apply_burst -from .burst_shared import apply as apply_burst_shared -from .chars import apply as apply_chars -from .discord_emojis import apply as apply_discord_emojis -from .duplicates import apply as apply_duplicates -from .links import apply as apply_links -from .mentions import apply as apply_mentions -from .newlines import apply as apply_newlines -from .role_mentions import apply as apply_role_mentions diff --git a/bot/rules/attachments.py b/bot/rules/attachments.py deleted file mode 100644 index 8903c385cb..0000000000 --- a/bot/rules/attachments.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects total attachments exceeding the limit sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if ( - msg.author == last_message.author - and len(msg.attachments) > 0 - ) - ) - total_recent_attachments = sum(len(msg.attachments) for msg in relevant_messages) - - if total_recent_attachments > config['max']: - return ( - f"sent {total_recent_attachments} attachments in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/bot/rules/burst.py b/bot/rules/burst.py deleted file mode 100644 index 25c5a2f335..0000000000 --- a/bot/rules/burst.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects repeated messages sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if msg.author == last_message.author - ) - total_relevant = len(relevant_messages) - - if total_relevant > config['max']: - return ( - f"sent {total_relevant} messages in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/bot/rules/burst_shared.py b/bot/rules/burst_shared.py deleted file mode 100644 index bbe9271b3b..0000000000 --- a/bot/rules/burst_shared.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects repeated messages sent by multiple users.""" - total_recent = len(recent_messages) - - if total_recent > config['max']: - return ( - f"sent {total_recent} messages in {config['interval']}s", - set(msg.author for msg in recent_messages), - recent_messages - ) - return None diff --git a/bot/rules/chars.py b/bot/rules/chars.py deleted file mode 100644 index 1f587422c5..0000000000 --- a/bot/rules/chars.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects total message char count exceeding the limit sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if msg.author == last_message.author - ) - - total_recent_chars = sum(len(msg.content) for msg in relevant_messages) - - if total_recent_chars > config['max']: - return ( - f"sent {total_recent_chars} characters in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/bot/rules/discord_emojis.py b/bot/rules/discord_emojis.py deleted file mode 100644 index d979ac5e77..0000000000 --- a/bot/rules/discord_emojis.py +++ /dev/null @@ -1,34 +0,0 @@ -import re -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message -from emoji import demojize - -DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>|:\w+:") -CODE_BLOCK_RE = re.compile(r"```.*?```", flags=re.DOTALL) - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects total Discord emojis exceeding the limit sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if msg.author == last_message.author - ) - - # Get rid of code blocks in the message before searching for emojis. - # Convert Unicode emojis to :emoji: format to get their count. - total_emojis = sum( - len(DISCORD_EMOJI_RE.findall(demojize(CODE_BLOCK_RE.sub("", msg.content)))) - for msg in relevant_messages - ) - - if total_emojis > config['max']: - return ( - f"sent {total_emojis} emojis in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/bot/rules/duplicates.py b/bot/rules/duplicates.py deleted file mode 100644 index 8e4fbc12df..0000000000 --- a/bot/rules/duplicates.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects duplicated messages sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if ( - msg.author == last_message.author - and msg.content == last_message.content - and msg.content - ) - ) - - total_duplicated = len(relevant_messages) - - if total_duplicated > config['max']: - return ( - f"sent {total_duplicated} duplicated messages in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/bot/rules/links.py b/bot/rules/links.py deleted file mode 100644 index c46b783c56..0000000000 --- a/bot/rules/links.py +++ /dev/null @@ -1,36 +0,0 @@ -import re -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - -LINK_RE = re.compile(r"(https?://[^\s]+)") - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects total links exceeding the limit sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if msg.author == last_message.author - ) - total_links = 0 - messages_with_links = 0 - - for msg in relevant_messages: - total_matches = len(LINK_RE.findall(msg.content)) - if total_matches: - messages_with_links += 1 - total_links += total_matches - - # Only apply the filter if we found more than one message with - # links to prevent wrongfully firing the rule on users posting - # e.g. an installation log of pip packages from GitHub. - if total_links > config['max'] and messages_with_links > 1: - return ( - f"sent {total_links} links in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/bot/rules/mentions.py b/bot/rules/mentions.py deleted file mode 100644 index 6f5addad1f..0000000000 --- a/bot/rules/mentions.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects total mentions exceeding the limit sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if msg.author == last_message.author - ) - - total_recent_mentions = sum( - not user.bot - for msg in relevant_messages - for user in msg.mentions - ) - - if total_recent_mentions > config['max']: - return ( - f"sent {total_recent_mentions} mentions in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/bot/rules/newlines.py b/bot/rules/newlines.py deleted file mode 100644 index 4e66e1359b..0000000000 --- a/bot/rules/newlines.py +++ /dev/null @@ -1,45 +0,0 @@ -import re -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects total newlines exceeding the set limit sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if msg.author == last_message.author - ) - - # Identify groups of newline characters and get group & total counts - exp = r"(\n+)" - newline_counts = [] - for msg in relevant_messages: - newline_counts += [len(group) for group in re.findall(exp, msg.content)] - total_recent_newlines = sum(newline_counts) - - # Get maximum newline group size - if newline_counts: - max_newline_group = max(newline_counts) - else: - # If no newlines are found, newline_counts will be an empty list, which will error out max() - max_newline_group = 0 - - # Check first for total newlines, if this passes then check for large groupings - if total_recent_newlines > config['max']: - return ( - f"sent {total_recent_newlines} newlines in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - elif max_newline_group > config['max_consecutive']: - return ( - f"sent {max_newline_group} consecutive newlines in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - - return None diff --git a/bot/rules/role_mentions.py b/bot/rules/role_mentions.py deleted file mode 100644 index 0649540b68..0000000000 --- a/bot/rules/role_mentions.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import Dict, Iterable, List, Optional, Tuple - -from discord import Member, Message - - -async def apply( - last_message: Message, recent_messages: List[Message], config: Dict[str, int] -) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects total role mentions exceeding the limit sent by a single user.""" - relevant_messages = tuple( - msg - for msg in recent_messages - if msg.author == last_message.author - ) - - total_recent_mentions = sum(len(msg.role_mentions) for msg in relevant_messages) - - if total_recent_mentions > config['max']: - return ( - f"sent {total_recent_mentions} role mentions in {config['interval']}s", - (last_message.author,), - relevant_messages - ) - return None diff --git a/tests/bot/exts/filters/__init__.py b/tests/bot/exts/filters/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/bot/exts/filters/test_antimalware.py b/tests/bot/exts/filters/test_antimalware.py deleted file mode 100644 index 7282334e2e..0000000000 --- a/tests/bot/exts/filters/test_antimalware.py +++ /dev/null @@ -1,202 +0,0 @@ -import unittest -from unittest.mock import AsyncMock, Mock - -from discord import NotFound - -from bot.constants import Channels, STAFF_ROLES -from bot.exts.filters import antimalware -from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole - - -class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): - """Test the AntiMalware cog.""" - - def setUp(self): - """Sets up fresh objects for each test.""" - self.bot = MockBot() - self.bot.filter_list_cache = { - "FILE_FORMAT.True": { - ".first": {}, - ".second": {}, - ".third": {}, - } - } - self.cog = antimalware.AntiMalware(self.bot) - self.message = MockMessage() - self.message.webhook_id = None - self.message.author.bot = None - self.whitelist = [".first", ".second", ".third"] - - async def test_message_with_allowed_attachment(self): - """Messages with allowed extensions should not be deleted""" - attachment = MockAttachment(filename="python.first") - self.message.attachments = [attachment] - - await self.cog.on_message(self.message) - self.message.delete.assert_not_called() - - async def test_message_without_attachment(self): - """Messages without attachments should result in no action.""" - await self.cog.on_message(self.message) - self.message.delete.assert_not_called() - - async def test_direct_message_with_attachment(self): - """Direct messages should have no action taken.""" - attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] - self.message.guild = None - - await self.cog.on_message(self.message) - - self.message.delete.assert_not_called() - - async def test_webhook_message_with_illegal_extension(self): - """A webhook message containing an illegal extension should be ignored.""" - attachment = MockAttachment(filename="python.disallowed") - self.message.webhook_id = 697140105563078727 - self.message.attachments = [attachment] - - await self.cog.on_message(self.message) - - self.message.delete.assert_not_called() - - async def test_bot_message_with_illegal_extension(self): - """A bot message containing an illegal extension should be ignored.""" - attachment = MockAttachment(filename="python.disallowed") - self.message.author.bot = 409107086526644234 - self.message.attachments = [attachment] - - await self.cog.on_message(self.message) - - self.message.delete.assert_not_called() - - async def test_message_with_illegal_extension_gets_deleted(self): - """A message containing an illegal extension should send an embed.""" - attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] - - await self.cog.on_message(self.message) - - self.message.delete.assert_called_once() - - async def test_message_send_by_staff(self): - """A message send by a member of staff should be ignored.""" - staff_role = MockRole(id=STAFF_ROLES[0]) - self.message.author.roles.append(staff_role) - attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] - - await self.cog.on_message(self.message) - - self.message.delete.assert_not_called() - - async def test_python_file_redirect_embed_description(self): - """A message containing a .py file should result in an embed redirecting the user to our paste site""" - attachment = MockAttachment(filename="python.py") - self.message.attachments = [attachment] - self.message.channel.send = AsyncMock() - - await self.cog.on_message(self.message) - self.message.channel.send.assert_called_once() - args, kwargs = self.message.channel.send.call_args - embed = kwargs.pop("embed") - - self.assertEqual(embed.description, antimalware.PY_EMBED_DESCRIPTION) - - async def test_txt_file_redirect_embed_description(self): - """A message containing a .txt/.json/.csv file should result in the correct embed.""" - test_values = ( - ("text", ".txt"), - ("json", ".json"), - ("csv", ".csv"), - ) - - for file_name, disallowed_extension in test_values: - with self.subTest(file_name=file_name, disallowed_extension=disallowed_extension): - - attachment = MockAttachment(filename=f"{file_name}{disallowed_extension}") - self.message.attachments = [attachment] - self.message.channel.send = AsyncMock() - antimalware.TXT_EMBED_DESCRIPTION = Mock() - antimalware.TXT_EMBED_DESCRIPTION.format.return_value = "test" - - await self.cog.on_message(self.message) - self.message.channel.send.assert_called_once() - args, kwargs = self.message.channel.send.call_args - embed = kwargs.pop("embed") - cmd_channel = self.bot.get_channel(Channels.bot_commands) - - self.assertEqual( - embed.description, - antimalware.TXT_EMBED_DESCRIPTION.format.return_value - ) - antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with( - blocked_extension=disallowed_extension, - cmd_channel_mention=cmd_channel.mention - ) - - async def test_other_disallowed_extension_embed_description(self): - """Test the description for a non .py/.txt/.json/.csv disallowed extension.""" - attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] - self.message.channel.send = AsyncMock() - antimalware.DISALLOWED_EMBED_DESCRIPTION = Mock() - antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value = "test" - - await self.cog.on_message(self.message) - self.message.channel.send.assert_called_once() - args, kwargs = self.message.channel.send.call_args - embed = kwargs.pop("embed") - meta_channel = self.bot.get_channel(Channels.meta) - - self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value) - antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( - joined_whitelist=", ".join(self.whitelist), - blocked_extensions_str=".disallowed", - meta_channel_mention=meta_channel.mention - ) - - async def test_removing_deleted_message_logs(self): - """Removing an already deleted message logs the correct message""" - attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] - self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - - with self.assertLogs(logger=antimalware.log, level="INFO"): - await self.cog.on_message(self.message) - self.message.delete.assert_called_once() - - async def test_message_with_illegal_attachment_logs(self): - """Deleting a message with an illegal attachment should result in a log.""" - attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] - - with self.assertLogs(logger=antimalware.log, level="INFO"): - await self.cog.on_message(self.message) - - async def test_get_disallowed_extensions(self): - """The return value should include all non-whitelisted extensions.""" - test_values = ( - ([], []), - (self.whitelist, []), - ([".first"], []), - ([".first", ".disallowed"], [".disallowed"]), - ([".disallowed"], [".disallowed"]), - ([".disallowed", ".illegal"], [".disallowed", ".illegal"]), - ) - - for extensions, expected_disallowed_extensions in test_values: - with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): - self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] - disallowed_extensions = self.cog._get_disallowed_extensions(self.message) - self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) - - -class AntiMalwareSetupTests(unittest.IsolatedAsyncioTestCase): - """Tests setup of the `AntiMalware` cog.""" - - async def test_setup(self): - """Setup of the extension should call add_cog.""" - bot = MockBot() - await antimalware.setup(bot) - bot.add_cog.assert_awaited_once() diff --git a/tests/bot/exts/filters/test_antispam.py b/tests/bot/exts/filters/test_antispam.py deleted file mode 100644 index 6a0e4fded1..0000000000 --- a/tests/bot/exts/filters/test_antispam.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest - -from bot.exts.filters import antispam - - -class AntispamConfigurationValidationTests(unittest.TestCase): - """Tests validation of the antispam cog configuration.""" - - def test_default_antispam_config_is_valid(self): - """The default antispam configuration is valid.""" - validation_errors = antispam.validate_config() - self.assertEqual(validation_errors, {}) - - def test_unknown_rule_returns_error(self): - """Configuring an unknown rule returns an error.""" - self.assertEqual( - antispam.validate_config({'invalid-rule': {}}), - {'invalid-rule': "`invalid-rule` is not recognized as an antispam rule."} - ) - - def test_missing_keys_returns_error(self): - """Not configuring required keys returns an error.""" - keys = (('interval', 'max'), ('max', 'interval')) - for configured_key, unconfigured_key in keys: - with self.subTest( - configured_key=configured_key, - unconfigured_key=unconfigured_key - ): - config = {'burst': {configured_key: 10}} - error = f"Key `{unconfigured_key}` is required but not set for rule `burst`" - - self.assertEqual( - antispam.validate_config(config), - {'burst': error} - ) diff --git a/tests/bot/exts/filters/test_filtering.py b/tests/bot/exts/filters/test_filtering.py deleted file mode 100644 index bd26532f19..0000000000 --- a/tests/bot/exts/filters/test_filtering.py +++ /dev/null @@ -1,40 +0,0 @@ -import unittest -from unittest.mock import patch - -from bot.exts.filters import filtering -from tests.helpers import MockBot, autospec - - -class FilteringCogTests(unittest.IsolatedAsyncioTestCase): - """Tests the `Filtering` cog.""" - - def setUp(self): - """Instantiate the bot and cog.""" - self.bot = MockBot() - with patch("botcore.utils.scheduling.create_task", new=lambda task, **_: task.close()): - self.cog = filtering.Filtering(self.bot) - - @autospec(filtering.Filtering, "_get_filterlist_items", pass_mocks=False, return_value=["TOKEN"]) - async def test_token_filter(self): - """Ensure that a filter token is correctly detected in a message.""" - messages = { - "": False, - "no matches": False, - "TOKEN": True, - - # See advisory https://github.com/python-discord/bot/security/advisories/GHSA-j8c3-8x46-8pp6 - "https://google.com TOKEN": True, - "https://google.com something else": False, - } - - for message, match in messages.items(): - with self.subTest(input=message, match=match): - result, _ = await self.cog._has_watch_regex_match(message) - - self.assertEqual( - match, - bool(result), - msg=f"Hit was {'expected' if match else 'not expected'} for this input." - ) - if result: - self.assertEqual("TOKEN", result.group()) diff --git a/tests/bot/exts/filters/test_security.py b/tests/bot/exts/filters/test_security.py deleted file mode 100644 index 007b7b1eb1..0000000000 --- a/tests/bot/exts/filters/test_security.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest - -from discord.ext.commands import NoPrivateMessage - -from bot.exts.filters import security -from tests.helpers import MockBot, MockContext - - -class SecurityCogTests(unittest.TestCase): - """Tests the `Security` cog.""" - - def setUp(self): - """Attach an instance of the cog to the class for tests.""" - self.bot = MockBot() - self.cog = security.Security(self.bot) - self.ctx = MockContext() - - def test_check_additions(self): - """The cog should add its checks after initialization.""" - self.bot.check.assert_any_call(self.cog.check_on_guild) - self.bot.check.assert_any_call(self.cog.check_not_bot) - - def test_check_not_bot_returns_false_for_humans(self): - """The bot check should return `True` when invoked with human authors.""" - self.ctx.author.bot = False - self.assertTrue(self.cog.check_not_bot(self.ctx)) - - def test_check_not_bot_returns_true_for_robots(self): - """The bot check should return `False` when invoked with robotic authors.""" - self.ctx.author.bot = True - self.assertFalse(self.cog.check_not_bot(self.ctx)) - - def test_check_on_guild_raises_when_outside_of_guild(self): - """When invoked outside of a guild, `check_on_guild` should cause an error.""" - self.ctx.guild = None - - with self.assertRaises(NoPrivateMessage, msg="This command cannot be used in private messages."): - self.cog.check_on_guild(self.ctx) - - def test_check_on_guild_returns_true_inside_of_guild(self): - """When invoked inside of a guild, `check_on_guild` should return `True`.""" - self.ctx.guild = "lemon's lemonade stand" - self.assertTrue(self.cog.check_on_guild(self.ctx)) - - -class SecurityCogLoadTests(unittest.IsolatedAsyncioTestCase): - """Tests loading the `Security` cog.""" - - async def test_security_cog_load(self): - """Setup of the extension should call add_cog.""" - bot = MockBot() - await security.setup(bot) - bot.add_cog.assert_awaited_once() diff --git a/tests/bot/exts/filters/test_token_remover.py b/tests/bot/exts/filters/test_token_remover.py deleted file mode 100644 index c1f3762ac1..0000000000 --- a/tests/bot/exts/filters/test_token_remover.py +++ /dev/null @@ -1,409 +0,0 @@ -import unittest -from re import Match -from unittest import mock -from unittest.mock import MagicMock - -from discord import Colour, NotFound - -from bot import constants -from bot.exts.filters import token_remover -from bot.exts.filters.token_remover import Token, TokenRemover -from bot.exts.moderation.modlog import ModLog -from bot.utils.messages import format_user -from tests.helpers import MockBot, MockMessage, autospec - - -class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): - """Tests the `TokenRemover` cog.""" - - def setUp(self): - """Adds the cog, a bot, and a message to the instance for usage in tests.""" - self.bot = MockBot() - self.cog = TokenRemover(bot=self.bot) - - self.msg = MockMessage(id=555, content="hello world") - self.msg.channel.mention = "#lemonade-stand" - self.msg.guild.get_member.return_value.bot = False - self.msg.guild.get_member.return_value.__str__.return_value = "Woody" - self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) - self.msg.author.display_avatar.url = "picture-lemon.png" - - def test_extract_user_id_valid(self): - """Should consider user IDs valid if they decode into an integer ID.""" - id_pairs = ( - ("NDcyMjY1OTQzMDYyNDEzMzMy", 472265943062413332), - ("NDc1MDczNjI5Mzk5NTQ3OTA0", 475073629399547904), - ("NDY3MjIzMjMwNjUwNzc3NjQx", 467223230650777641), - ) - - for token_id, user_id in id_pairs: - with self.subTest(token_id=token_id): - result = TokenRemover.extract_user_id(token_id) - self.assertEqual(result, user_id) - - def test_extract_user_id_invalid(self): - """Should consider non-digit and non-ASCII IDs invalid.""" - ids = ( - ("SGVsbG8gd29ybGQ", "non-digit ASCII"), - ("0J_RgNC40LLQtdGCINC80LjRgA", "cyrillic text"), - ("4pO14p6L4p6C4pG34p264pGl8J-EiOKSj-KCieKBsA", "Unicode digits"), - ("4oaA4oaB4oWh4oWi4Lyz4Lyq4Lyr4LG9", "Unicode numerals"), - ("8J2fjvCdn5nwnZ-k8J2fr_Cdn7rgravvvJngr6c", "Unicode decimals"), - ("{hello}[world]&(bye!)", "ASCII invalid Base64"), - ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), - ) - - for user_id, msg in ids: - with self.subTest(msg=msg): - result = TokenRemover.extract_user_id(user_id) - self.assertIsNone(result) - - def test_is_valid_timestamp_valid(self): - """Should consider timestamps valid if they're greater than the Discord epoch.""" - timestamps = ( - "XsyRkw", - "Xrim9Q", - "XsyR-w", - "XsySD_", - "Dn9r_A", - ) - - for timestamp in timestamps: - with self.subTest(timestamp=timestamp): - result = TokenRemover.is_valid_timestamp(timestamp) - self.assertTrue(result) - - def test_is_valid_timestamp_invalid(self): - """Should consider timestamps invalid if they're before Discord epoch or can't be parsed.""" - timestamps = ( - ("B4Yffw", "DISCORD_EPOCH - TOKEN_EPOCH - 1"), - ("ew", "123"), - ("AoIKgA", "42076800"), - ("{hello}[world]&(bye!)", "ASCII invalid Base64"), - ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), - ) - - for timestamp, msg in timestamps: - with self.subTest(msg=msg): - result = TokenRemover.is_valid_timestamp(timestamp) - self.assertFalse(result) - - def test_is_valid_hmac_valid(self): - """Should consider an HMAC valid if it has at least 3 unique characters.""" - valid_hmacs = ( - "VXmErH7j511turNpfURmb0rVNm8", - "Ysnu2wacjaKs7qnoo46S8Dm2us8", - "sJf6omBPORBPju3WJEIAcwW9Zds", - "s45jqDV_Iisn-symw0yDRrk_jf4", - ) - - for hmac in valid_hmacs: - with self.subTest(msg=hmac): - result = TokenRemover.is_maybe_valid_hmac(hmac) - self.assertTrue(result) - - def test_is_invalid_hmac_invalid(self): - """Should consider an HMAC invalid if has fewer than 3 unique characters.""" - invalid_hmacs = ( - ("xxxxxxxxxxxxxxxxxx", "Single character"), - ("XxXxXxXxXxXxXxXxXx", "Single character alternating case"), - ("ASFasfASFasfASFASsf", "Three characters alternating-case"), - ("asdasdasdasdasdasdasd", "Three characters one case"), - ) - - for hmac, msg in invalid_hmacs: - with self.subTest(msg=msg): - result = TokenRemover.is_maybe_valid_hmac(hmac) - self.assertFalse(result) - - def test_mod_log_property(self): - """The `mod_log` property should ask the bot to return the `ModLog` cog.""" - self.bot.get_cog.return_value = 'lemon' - self.assertEqual(self.cog.mod_log, self.bot.get_cog.return_value) - self.bot.get_cog.assert_called_once_with('ModLog') - - async def test_on_message_edit_uses_on_message(self): - """The edit listener should delegate handling of the message to the normal listener.""" - self.cog.on_message = mock.create_autospec(self.cog.on_message, spec_set=True) - - await self.cog.on_message_edit(MockMessage(), self.msg) - self.cog.on_message.assert_awaited_once_with(self.msg) - - @autospec(TokenRemover, "find_token_in_message", "take_action") - async def test_on_message_takes_action(self, find_token_in_message, take_action): - """Should take action if a valid token is found when a message is sent.""" - cog = TokenRemover(self.bot) - found_token = "foobar" - find_token_in_message.return_value = found_token - - await cog.on_message(self.msg) - - find_token_in_message.assert_called_once_with(self.msg) - take_action.assert_awaited_once_with(cog, self.msg, found_token) - - @autospec(TokenRemover, "find_token_in_message", "take_action") - async def test_on_message_skips_missing_token(self, find_token_in_message, take_action): - """Shouldn't take action if a valid token isn't found when a message is sent.""" - cog = TokenRemover(self.bot) - find_token_in_message.return_value = False - - await cog.on_message(self.msg) - - find_token_in_message.assert_called_once_with(self.msg) - take_action.assert_not_awaited() - - @autospec(TokenRemover, "find_token_in_message") - async def test_on_message_ignores_dms_bots(self, find_token_in_message): - """Shouldn't parse a message if it is a DM or authored by a bot.""" - cog = TokenRemover(self.bot) - dm_msg = MockMessage(guild=None) - bot_msg = MockMessage(author=MagicMock(bot=True)) - - for msg in (dm_msg, bot_msg): - await cog.on_message(msg) - find_token_in_message.assert_not_called() - - @autospec("bot.exts.filters.token_remover", "TOKEN_RE") - def test_find_token_no_matches(self, token_re): - """None should be returned if the regex matches no tokens in a message.""" - token_re.finditer.return_value = () - - return_value = TokenRemover.find_token_in_message(self.msg) - - self.assertIsNone(return_value) - token_re.finditer.assert_called_once_with(self.msg.content) - - @autospec(TokenRemover, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac") - @autospec("bot.exts.filters.token_remover", "Token") - @autospec("bot.exts.filters.token_remover", "TOKEN_RE") - def test_find_token_valid_match( - self, - token_re, - token_cls, - extract_user_id, - is_valid_timestamp, - is_maybe_valid_hmac, - ): - """The first match with a valid user ID, timestamp, and HMAC should be returned as a `Token`.""" - matches = [ - mock.create_autospec(Match, spec_set=True, instance=True), - mock.create_autospec(Match, spec_set=True, instance=True), - ] - tokens = [ - mock.create_autospec(Token, spec_set=True, instance=True), - mock.create_autospec(Token, spec_set=True, instance=True), - ] - - token_re.finditer.return_value = matches - token_cls.side_effect = tokens - extract_user_id.side_effect = (None, True) # The 1st match will be invalid, 2nd one valid. - is_valid_timestamp.return_value = True - is_maybe_valid_hmac.return_value = True - - return_value = TokenRemover.find_token_in_message(self.msg) - - self.assertEqual(tokens[1], return_value) - token_re.finditer.assert_called_once_with(self.msg.content) - - @autospec(TokenRemover, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac") - @autospec("bot.exts.filters.token_remover", "Token") - @autospec("bot.exts.filters.token_remover", "TOKEN_RE") - def test_find_token_invalid_matches( - self, - token_re, - token_cls, - extract_user_id, - is_valid_timestamp, - is_maybe_valid_hmac, - ): - """None should be returned if no matches have valid user IDs, HMACs, and timestamps.""" - token_re.finditer.return_value = [mock.create_autospec(Match, spec_set=True, instance=True)] - token_cls.return_value = mock.create_autospec(Token, spec_set=True, instance=True) - extract_user_id.return_value = None - is_valid_timestamp.return_value = False - is_maybe_valid_hmac.return_value = False - - return_value = TokenRemover.find_token_in_message(self.msg) - - self.assertIsNone(return_value) - token_re.finditer.assert_called_once_with(self.msg.content) - - def test_regex_invalid_tokens(self): - """Messages without anything looking like a token are not matched.""" - tokens = ( - "", - "lemon wins", - "..", - "x.y", - "x.y.", - ".y.z", - ".y.", - "..z", - "x..z", - " . . ", - "\n.\n.\n", - "hellö.world.bye", - "base64.nötbåse64.morebase64", - "19jd3J.dfkm3d.€víł§tüff", - ) - - for token in tokens: - with self.subTest(token=token): - results = token_remover.TOKEN_RE.findall(token) - self.assertEqual(len(results), 0) - - def test_regex_valid_tokens(self): - """Messages that look like tokens should be matched.""" - # Don't worry, these tokens have been invalidated. - tokens = ( - "NDcyMjY1OTQzMDYy_DEzMz-y.XsyRkw.VXmErH7j511turNpfURmb0rVNm8", - "NDcyMjY1OTQzMDYyNDEzMzMy.Xrim9Q.Ysnu2wacjaKs7qnoo46S8Dm2us8", - "NDc1MDczNjI5Mzk5NTQ3OTA0.XsyR-w.sJf6omBPORBPju3WJEIAcwW9Zds", - "NDY3MjIzMjMwNjUwNzc3NjQx.XsySD_.s45jqDV_Iisn-symw0yDRrk_jf4", - ) - - for token in tokens: - with self.subTest(token=token): - results = token_remover.TOKEN_RE.fullmatch(token) - self.assertIsNotNone(results, f"{token} was not matched by the regex") - - def test_regex_matches_multiple_valid(self): - """Should support multiple matches in the middle of a string.""" - token_1 = "NDY3MjIzMjMwNjUwNzc3NjQx.XsyWGg.uFNEQPCc4ePwGh7egG8UicQssz8" - token_2 = "NDcyMjY1OTQzMDYyNDEzMzMy.XsyWMw.l8XPnDqb0lp-EiQ2g_0xVFT1pyc" - message = f"garbage {token_1} hello {token_2} world" - - results = token_remover.TOKEN_RE.finditer(message) - results = [match[0] for match in results] - self.assertCountEqual((token_1, token_2), results) - - @autospec("bot.exts.filters.token_remover", "LOG_MESSAGE") - def test_format_log_message(self, log_message): - """Should correctly format the log message with info from the message and token.""" - token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") - log_message.format.return_value = "Howdy" - - return_value = TokenRemover.format_log_message(self.msg, token) - - self.assertEqual(return_value, log_message.format.return_value) - log_message.format.assert_called_once_with( - author=format_user(self.msg.author), - channel=self.msg.channel.mention, - user_id=token.user_id, - timestamp=token.timestamp, - hmac="xxxxxxxxxxxxxxxxxxxxxxxxjf4", - ) - - @autospec("bot.exts.filters.token_remover", "UNKNOWN_USER_LOG_MESSAGE") - async def test_format_userid_log_message_unknown(self, unknown_user_log_message,): - """Should correctly format the user ID portion when the actual user it belongs to is unknown.""" - token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") - unknown_user_log_message.format.return_value = " Partner" - msg = MockMessage(id=555, content="hello world") - msg.guild.get_member.return_value = None - msg.guild.fetch_member.side_effect = NotFound(mock.Mock(status=404), "Not found") - - return_value = await TokenRemover.format_userid_log_message(msg, token) - - self.assertEqual(return_value, (unknown_user_log_message.format.return_value, False)) - unknown_user_log_message.format.assert_called_once_with(user_id=472265943062413332) - - @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") - async def test_format_userid_log_message_bot(self, known_user_log_message): - """Should correctly format the user ID portion when the ID belongs to a known bot.""" - token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") - known_user_log_message.format.return_value = " Partner" - msg = MockMessage(id=555, content="hello world") - msg.guild.get_member.return_value.__str__.return_value = "Sam" - msg.guild.get_member.return_value.bot = True - - return_value = await TokenRemover.format_userid_log_message(msg, token) - - self.assertEqual(return_value, (known_user_log_message.format.return_value, True)) - - known_user_log_message.format.assert_called_once_with( - user_id=472265943062413332, - user_name="Sam", - kind="BOT", - ) - - @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") - async def test_format_log_message_user_token_user(self, user_token_message): - """Should correctly format the user ID portion when the ID belongs to a known user.""" - token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") - user_token_message.format.return_value = "Partner" - - return_value = await TokenRemover.format_userid_log_message(self.msg, token) - - self.assertEqual(return_value, (user_token_message.format.return_value, True)) - user_token_message.format.assert_called_once_with( - user_id=467223230650777641, - user_name="Woody", - kind="USER", - ) - - @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) - @autospec("bot.exts.filters.token_remover", "log") - @autospec(TokenRemover, "format_log_message", "format_userid_log_message") - async def test_take_action(self, format_log_message, format_userid_log_message, logger, mod_log_property): - """Should delete the message and send a mod log.""" - cog = TokenRemover(self.bot) - mod_log = mock.create_autospec(ModLog, spec_set=True, instance=True) - token = mock.create_autospec(Token, spec_set=True, instance=True) - token.user_id = "no-id" - log_msg = "testing123" - userid_log_message = "userid-log-message" - - mod_log_property.return_value = mod_log - format_log_message.return_value = log_msg - format_userid_log_message.return_value = (userid_log_message, True) - - await cog.take_action(self.msg, token) - - self.msg.delete.assert_called_once_with() - self.msg.channel.send.assert_called_once_with( - token_remover.DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) - ) - - format_log_message.assert_called_once_with(self.msg, token) - format_userid_log_message.assert_called_once_with(self.msg, token) - logger.debug.assert_called_with(log_msg) - self.bot.stats.incr.assert_called_once_with("tokens.removed_tokens") - - mod_log.ignore.assert_called_once_with(constants.Event.message_delete, self.msg.id) - mod_log.send_log_message.assert_called_once_with( - icon_url=constants.Icons.token_removed, - colour=Colour(constants.Colours.soft_red), - title="Token removed!", - text=log_msg + "\n" + userid_log_message, - thumbnail=self.msg.author.display_avatar.url, - channel_id=constants.Channels.mod_alerts, - ping_everyone=True, - ) - - @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) - async def test_take_action_delete_failure(self, mod_log_property): - """Shouldn't send any messages if the token message can't be deleted.""" - cog = TokenRemover(self.bot) - mod_log_property.return_value = mock.create_autospec(ModLog, spec_set=True, instance=True) - self.msg.delete.side_effect = NotFound(MagicMock(), MagicMock()) - - token = mock.create_autospec(Token, spec_set=True, instance=True) - await cog.take_action(self.msg, token) - - self.msg.delete.assert_called_once_with() - self.msg.channel.send.assert_not_awaited() - - -class TokenRemoverExtensionTests(unittest.IsolatedAsyncioTestCase): - """Tests for the token_remover extension.""" - - @autospec("bot.exts.filters.token_remover", "TokenRemover") - async def test_extension_setup(self, cog): - """The TokenRemover cog should be added.""" - bot = MockBot() - await token_remover.setup(bot) - - cog.assert_called_once_with(bot) - bot.add_cog.assert_awaited_once() - self.assertTrue(isinstance(bot.add_cog.call_args.args[0], TokenRemover)) diff --git a/tests/bot/rules/__init__.py b/tests/bot/rules/__init__.py deleted file mode 100644 index 0d570f5a3d..0000000000 --- a/tests/bot/rules/__init__.py +++ /dev/null @@ -1,76 +0,0 @@ -import unittest -from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterable, List, NamedTuple, Tuple - -from tests.helpers import MockMessage - - -class DisallowedCase(NamedTuple): - """Encapsulation for test cases expected to fail.""" - recent_messages: List[MockMessage] - culprits: Iterable[str] - n_violations: int - - -class RuleTest(unittest.IsolatedAsyncioTestCase, metaclass=ABCMeta): - """ - Abstract class for antispam rule test cases. - - Tests for specific rules should inherit from `RuleTest` and implement - `relevant_messages` and `get_report`. Each instance should also set the - `apply` and `config` attributes as necessary. - - The execution of test cases can then be delegated to the `run_allowed` - and `run_disallowed` methods. - """ - - apply: Callable # The tested rule's apply function - config: Dict[str, int] - - async def run_allowed(self, cases: Tuple[List[MockMessage], ...]) -> None: - """Run all `cases` against `self.apply` expecting them to pass.""" - for recent_messages in cases: - last_message = recent_messages[0] - - with self.subTest( - last_message=last_message, - recent_messages=recent_messages, - config=self.config, - ): - self.assertIsNone( - await self.apply(last_message, recent_messages, self.config) - ) - - async def run_disallowed(self, cases: Tuple[DisallowedCase, ...]) -> None: - """Run all `cases` against `self.apply` expecting them to fail.""" - for case in cases: - recent_messages, culprits, n_violations = case - last_message = recent_messages[0] - relevant_messages = self.relevant_messages(case) - desired_output = ( - self.get_report(case), - culprits, - relevant_messages, - ) - - with self.subTest( - last_message=last_message, - recent_messages=recent_messages, - relevant_messages=relevant_messages, - n_violations=n_violations, - config=self.config, - ): - self.assertTupleEqual( - await self.apply(last_message, recent_messages, self.config), - desired_output, - ) - - @abstractmethod - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - """Give expected relevant messages for `case`.""" - raise NotImplementedError # pragma: no cover - - @abstractmethod - def get_report(self, case: DisallowedCase) -> str: - """Give expected error report for `case`.""" - raise NotImplementedError # pragma: no cover diff --git a/tests/bot/rules/test_attachments.py b/tests/bot/rules/test_attachments.py deleted file mode 100644 index d7e7792214..0000000000 --- a/tests/bot/rules/test_attachments.py +++ /dev/null @@ -1,69 +0,0 @@ -from typing import Iterable - -from bot.rules import attachments -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str, total_attachments: int) -> MockMessage: - """Builds a message with `total_attachments` attachments.""" - return MockMessage(author=author, attachments=list(range(total_attachments))) - - -class AttachmentRuleTests(RuleTest): - """Tests applying the `attachments` antispam rule.""" - - def setUp(self): - self.apply = attachments.apply - self.config = {"max": 5, "interval": 10} - - async def test_allows_messages_without_too_many_attachments(self): - """Messages without too many attachments are allowed as-is.""" - cases = ( - [make_msg("bob", 0), make_msg("bob", 0), make_msg("bob", 0)], - [make_msg("bob", 2), make_msg("bob", 2)], - [make_msg("bob", 2), make_msg("alice", 2), make_msg("bob", 2)], - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_with_too_many_attachments(self): - """Messages with too many attachments trigger the rule.""" - cases = ( - DisallowedCase( - [make_msg("bob", 4), make_msg("bob", 0), make_msg("bob", 6)], - ("bob",), - 10, - ), - DisallowedCase( - [make_msg("bob", 4), make_msg("alice", 6), make_msg("bob", 2)], - ("bob",), - 6, - ), - DisallowedCase( - [make_msg("alice", 6)], - ("alice",), - 6, - ), - DisallowedCase( - [make_msg("alice", 1) for _ in range(6)], - ("alice",), - 6, - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_message = case.recent_messages[0] - return tuple( - msg - for msg in case.recent_messages - if ( - msg.author == last_message.author - and len(msg.attachments) > 0 - ) - ) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} attachments in {self.config['interval']}s" diff --git a/tests/bot/rules/test_burst.py b/tests/bot/rules/test_burst.py deleted file mode 100644 index 03682966b8..0000000000 --- a/tests/bot/rules/test_burst.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import Iterable - -from bot.rules import burst -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str) -> MockMessage: - """ - Init a MockMessage instance with author set to `author`. - - This serves as a shorthand / alias to keep the test cases visually clean. - """ - return MockMessage(author=author) - - -class BurstRuleTests(RuleTest): - """Tests the `burst` antispam rule.""" - - def setUp(self): - self.apply = burst.apply - self.config = {"max": 2, "interval": 10} - - async def test_allows_messages_within_limit(self): - """Cases which do not violate the rule.""" - cases = ( - [make_msg("bob"), make_msg("bob")], - [make_msg("bob"), make_msg("alice"), make_msg("bob")], - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_beyond_limit(self): - """Cases where the amount of messages exceeds the limit, triggering the rule.""" - cases = ( - DisallowedCase( - [make_msg("bob"), make_msg("bob"), make_msg("bob")], - ("bob",), - 3, - ), - DisallowedCase( - [make_msg("bob"), make_msg("bob"), make_msg("alice"), make_msg("bob")], - ("bob",), - 3, - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - return tuple(msg for msg in case.recent_messages if msg.author in case.culprits) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} messages in {self.config['interval']}s" diff --git a/tests/bot/rules/test_burst_shared.py b/tests/bot/rules/test_burst_shared.py deleted file mode 100644 index 3275143d5b..0000000000 --- a/tests/bot/rules/test_burst_shared.py +++ /dev/null @@ -1,57 +0,0 @@ -from typing import Iterable - -from bot.rules import burst_shared -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str) -> MockMessage: - """ - Init a MockMessage instance with the passed arg. - - This serves as a shorthand / alias to keep the test cases visually clean. - """ - return MockMessage(author=author) - - -class BurstSharedRuleTests(RuleTest): - """Tests the `burst_shared` antispam rule.""" - - def setUp(self): - self.apply = burst_shared.apply - self.config = {"max": 2, "interval": 10} - - async def test_allows_messages_within_limit(self): - """ - Cases that do not violate the rule. - - There really isn't more to test here than a single case. - """ - cases = ( - [make_msg("spongebob"), make_msg("patrick")], - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_beyond_limit(self): - """Cases where the amount of messages exceeds the limit, triggering the rule.""" - cases = ( - DisallowedCase( - [make_msg("bob"), make_msg("bob"), make_msg("bob")], - {"bob"}, - 3, - ), - DisallowedCase( - [make_msg("bob"), make_msg("bob"), make_msg("alice"), make_msg("bob")], - {"bob", "alice"}, - 4, - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - return case.recent_messages - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} messages in {self.config['interval']}s" diff --git a/tests/bot/rules/test_chars.py b/tests/bot/rules/test_chars.py deleted file mode 100644 index f1e3c76a77..0000000000 --- a/tests/bot/rules/test_chars.py +++ /dev/null @@ -1,64 +0,0 @@ -from typing import Iterable - -from bot.rules import chars -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str, n_chars: int) -> MockMessage: - """Build a message with arbitrary content of `n_chars` length.""" - return MockMessage(author=author, content="A" * n_chars) - - -class CharsRuleTests(RuleTest): - """Tests the `chars` antispam rule.""" - - def setUp(self): - self.apply = chars.apply - self.config = { - "max": 20, # Max allowed sum of chars per user - "interval": 10, - } - - async def test_allows_messages_within_limit(self): - """Cases with a total amount of chars within limit.""" - cases = ( - [make_msg("bob", 0)], - [make_msg("bob", 20)], - [make_msg("bob", 15), make_msg("alice", 15)], - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_beyond_limit(self): - """Cases where the total amount of chars exceeds the limit, triggering the rule.""" - cases = ( - DisallowedCase( - [make_msg("bob", 21)], - ("bob",), - 21, - ), - DisallowedCase( - [make_msg("bob", 15), make_msg("bob", 15)], - ("bob",), - 30, - ), - DisallowedCase( - [make_msg("alice", 15), make_msg("bob", 20), make_msg("alice", 15)], - ("alice",), - 30, - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_message = case.recent_messages[0] - return tuple( - msg - for msg in case.recent_messages - if msg.author == last_message.author - ) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} characters in {self.config['interval']}s" diff --git a/tests/bot/rules/test_discord_emojis.py b/tests/bot/rules/test_discord_emojis.py deleted file mode 100644 index 66c2d9f92d..0000000000 --- a/tests/bot/rules/test_discord_emojis.py +++ /dev/null @@ -1,73 +0,0 @@ -from typing import Iterable - -from bot.rules import discord_emojis -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - -discord_emoji = "<:abcd:1234>" # Discord emojis follow the format <:name:id> -unicode_emoji = "🧪" - - -def make_msg(author: str, n_emojis: int, emoji: str = discord_emoji) -> MockMessage: - """Build a MockMessage instance with content containing `n_emojis` arbitrary emojis.""" - return MockMessage(author=author, content=emoji * n_emojis) - - -class DiscordEmojisRuleTests(RuleTest): - """Tests for the `discord_emojis` antispam rule.""" - - def setUp(self): - self.apply = discord_emojis.apply - self.config = {"max": 2, "interval": 10} - - async def test_allows_messages_within_limit(self): - """Cases with a total amount of discord and unicode emojis within limit.""" - cases = ( - [make_msg("bob", 2)], - [make_msg("alice", 1), make_msg("bob", 2), make_msg("alice", 1)], - [make_msg("bob", 2, unicode_emoji)], - [ - make_msg("alice", 1, unicode_emoji), - make_msg("bob", 2, unicode_emoji), - make_msg("alice", 1, unicode_emoji) - ], - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_beyond_limit(self): - """Cases with more than the allowed amount of discord and unicode emojis.""" - cases = ( - DisallowedCase( - [make_msg("bob", 3)], - ("bob",), - 3, - ), - DisallowedCase( - [make_msg("alice", 2), make_msg("bob", 2), make_msg("alice", 2)], - ("alice",), - 4, - ), - DisallowedCase( - [make_msg("bob", 3, unicode_emoji)], - ("bob",), - 3, - ), - DisallowedCase( - [ - make_msg("alice", 2, unicode_emoji), - make_msg("bob", 2, unicode_emoji), - make_msg("alice", 2, unicode_emoji) - ], - ("alice",), - 4 - ) - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - return tuple(msg for msg in case.recent_messages if msg.author in case.culprits) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} emojis in {self.config['interval']}s" diff --git a/tests/bot/rules/test_duplicates.py b/tests/bot/rules/test_duplicates.py deleted file mode 100644 index 9bd886a779..0000000000 --- a/tests/bot/rules/test_duplicates.py +++ /dev/null @@ -1,64 +0,0 @@ -from typing import Iterable - -from bot.rules import duplicates -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str, content: str) -> MockMessage: - """Give a MockMessage instance with `author` and `content` attrs.""" - return MockMessage(author=author, content=content) - - -class DuplicatesRuleTests(RuleTest): - """Tests the `duplicates` antispam rule.""" - - def setUp(self): - self.apply = duplicates.apply - self.config = {"max": 2, "interval": 10} - - async def test_allows_messages_within_limit(self): - """Cases which do not violate the rule.""" - cases = ( - [make_msg("alice", "A"), make_msg("alice", "A")], - [make_msg("alice", "A"), make_msg("alice", "B"), make_msg("alice", "C")], # Non-duplicate - [make_msg("alice", "A"), make_msg("bob", "A"), make_msg("alice", "A")], # Different author - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_beyond_limit(self): - """Cases with too many duplicate messages from the same author.""" - cases = ( - DisallowedCase( - [make_msg("alice", "A"), make_msg("alice", "A"), make_msg("alice", "A")], - ("alice",), - 3, - ), - DisallowedCase( - [make_msg("bob", "A"), make_msg("alice", "A"), make_msg("bob", "A"), make_msg("bob", "A")], - ("bob",), - 3, # 4 duplicate messages, but only 3 from bob - ), - DisallowedCase( - [make_msg("bob", "A"), make_msg("bob", "B"), make_msg("bob", "A"), make_msg("bob", "A")], - ("bob",), - 3, # 4 message from bob, but only 3 duplicates - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_message = case.recent_messages[0] - return tuple( - msg - for msg in case.recent_messages - if ( - msg.author == last_message.author - and msg.content == last_message.content - ) - ) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} duplicated messages in {self.config['interval']}s" diff --git a/tests/bot/rules/test_links.py b/tests/bot/rules/test_links.py deleted file mode 100644 index b091bd9d72..0000000000 --- a/tests/bot/rules/test_links.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import Iterable - -from bot.rules import links -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str, total_links: int) -> MockMessage: - """Makes a message with `total_links` links.""" - content = " ".join(["https://pydis.com"] * total_links) - return MockMessage(author=author, content=content) - - -class LinksTests(RuleTest): - """Tests applying the `links` rule.""" - - def setUp(self): - self.apply = links.apply - self.config = { - "max": 2, - "interval": 10 - } - - async def test_links_within_limit(self): - """Messages with an allowed amount of links.""" - cases = ( - [make_msg("bob", 0)], - [make_msg("bob", 2)], - [make_msg("bob", 3)], # Filter only applies if len(messages_with_links) > 1 - [make_msg("bob", 1), make_msg("bob", 1)], - [make_msg("bob", 2), make_msg("alice", 2)] # Only messages from latest author count - ) - - await self.run_allowed(cases) - - async def test_links_exceeding_limit(self): - """Messages with a a higher than allowed amount of links.""" - cases = ( - DisallowedCase( - [make_msg("bob", 1), make_msg("bob", 2)], - ("bob",), - 3 - ), - DisallowedCase( - [make_msg("alice", 1), make_msg("alice", 1), make_msg("alice", 1)], - ("alice",), - 3 - ), - DisallowedCase( - [make_msg("alice", 2), make_msg("bob", 3), make_msg("alice", 1)], - ("alice",), - 3 - ) - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_message = case.recent_messages[0] - return tuple( - msg - for msg in case.recent_messages - if msg.author == last_message.author - ) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} links in {self.config['interval']}s" diff --git a/tests/bot/rules/test_mentions.py b/tests/bot/rules/test_mentions.py deleted file mode 100644 index f8805ac48a..0000000000 --- a/tests/bot/rules/test_mentions.py +++ /dev/null @@ -1,83 +0,0 @@ -from typing import Iterable - -from bot.rules import mentions -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMember, MockMessage - - -def make_msg(author: str, total_user_mentions: int, total_bot_mentions: int = 0) -> MockMessage: - """Makes a message with `total_mentions` mentions.""" - user_mentions = [MockMember() for _ in range(total_user_mentions)] - bot_mentions = [MockMember(bot=True) for _ in range(total_bot_mentions)] - return MockMessage(author=author, mentions=user_mentions+bot_mentions) - - -class TestMentions(RuleTest): - """Tests applying the `mentions` antispam rule.""" - - def setUp(self): - self.apply = mentions.apply - self.config = { - "max": 2, - "interval": 10, - } - - async def test_mentions_within_limit(self): - """Messages with an allowed amount of mentions.""" - cases = ( - [make_msg("bob", 0)], - [make_msg("bob", 2)], - [make_msg("bob", 1), make_msg("bob", 1)], - [make_msg("bob", 1), make_msg("alice", 2)], - ) - - await self.run_allowed(cases) - - async def test_mentions_exceeding_limit(self): - """Messages with a higher than allowed amount of mentions.""" - cases = ( - DisallowedCase( - [make_msg("bob", 3)], - ("bob",), - 3, - ), - DisallowedCase( - [make_msg("alice", 2), make_msg("alice", 0), make_msg("alice", 1)], - ("alice",), - 3, - ), - DisallowedCase( - [make_msg("bob", 2), make_msg("alice", 3), make_msg("bob", 2)], - ("bob",), - 4, - ), - DisallowedCase( - [make_msg("bob", 3, 1)], - ("bob",), - 3, - ), - ) - - await self.run_disallowed(cases) - - async def test_ignore_bot_mentions(self): - """Messages with an allowed amount of mentions, also containing bot mentions.""" - cases = ( - [make_msg("bob", 0, 3)], - [make_msg("bob", 2, 1)], - [make_msg("bob", 1, 2), make_msg("bob", 1, 2)], - [make_msg("bob", 1, 5), make_msg("alice", 2, 5)] - ) - - await self.run_allowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_message = case.recent_messages[0] - return tuple( - msg - for msg in case.recent_messages - if msg.author == last_message.author - ) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} mentions in {self.config['interval']}s" diff --git a/tests/bot/rules/test_newlines.py b/tests/bot/rules/test_newlines.py deleted file mode 100644 index e35377773e..0000000000 --- a/tests/bot/rules/test_newlines.py +++ /dev/null @@ -1,102 +0,0 @@ -from typing import Iterable, List - -from bot.rules import newlines -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str, newline_groups: List[int]) -> MockMessage: - """Init a MockMessage instance with `author` and content configured by `newline_groups". - - Configure content by passing a list of ints, where each int `n` will generate - a separate group of `n` newlines. - - Example: - newline_groups=[3, 1, 2] -> content="\n\n\n \n \n\n" - """ - content = " ".join("\n" * n for n in newline_groups) - return MockMessage(author=author, content=content) - - -class TotalNewlinesRuleTests(RuleTest): - """Tests the `newlines` antispam rule against allowed cases and total newline count violations.""" - - def setUp(self): - self.apply = newlines.apply - self.config = { - "max": 5, # Max sum of newlines in relevant messages - "max_consecutive": 3, # Max newlines in one group, in one message - "interval": 10, - } - - async def test_allows_messages_within_limit(self): - """Cases which do not violate the rule.""" - cases = ( - [make_msg("alice", [])], # Single message with no newlines - [make_msg("alice", [1, 2]), make_msg("alice", [1, 1])], # 5 newlines in 2 messages - [make_msg("alice", [2, 2, 1]), make_msg("bob", [2, 3])], # 5 newlines from each author - [make_msg("bob", [1]), make_msg("alice", [5])], # Alice breaks the rule, but only bob is relevant - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_total(self): - """Cases which violate the rule by having too many newlines in total.""" - cases = ( - DisallowedCase( # Alice sends a total of 6 newlines (disallowed) - [make_msg("alice", [2, 2]), make_msg("alice", [2])], - ("alice",), - 6, - ), - DisallowedCase( # Here we test that only alice's newlines count in the sum - [make_msg("alice", [2, 2]), make_msg("bob", [3]), make_msg("alice", [3])], - ("alice",), - 7, - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_author = case.recent_messages[0].author - return tuple(msg for msg in case.recent_messages if msg.author == last_author) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} newlines in {self.config['interval']}s" - - -class GroupNewlinesRuleTests(RuleTest): - """ - Tests the `newlines` antispam rule against max consecutive newline violations. - - As these violations yield a different error report, they require a different - `get_report` implementation. - """ - - def setUp(self): - self.apply = newlines.apply - self.config = {"max": 5, "max_consecutive": 3, "interval": 10} - - async def test_disallows_messages_consecutive(self): - """Cases which violate the rule due to having too many consecutive newlines.""" - cases = ( - DisallowedCase( # Bob sends a group of newlines too large - [make_msg("bob", [4])], - ("bob",), - 4, - ), - DisallowedCase( # Alice sends 5 in total (allowed), but 4 in one group (disallowed) - [make_msg("alice", [1]), make_msg("alice", [4])], - ("alice",), - 4, - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_author = case.recent_messages[0].author - return tuple(msg for msg in case.recent_messages if msg.author == last_author) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} consecutive newlines in {self.config['interval']}s" diff --git a/tests/bot/rules/test_role_mentions.py b/tests/bot/rules/test_role_mentions.py deleted file mode 100644 index 26c05d5277..0000000000 --- a/tests/bot/rules/test_role_mentions.py +++ /dev/null @@ -1,55 +0,0 @@ -from typing import Iterable - -from bot.rules import role_mentions -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage - - -def make_msg(author: str, n_mentions: int) -> MockMessage: - """Build a MockMessage instance with `n_mentions` role mentions.""" - return MockMessage(author=author, role_mentions=[None] * n_mentions) - - -class RoleMentionsRuleTests(RuleTest): - """Tests for the `role_mentions` antispam rule.""" - - def setUp(self): - self.apply = role_mentions.apply - self.config = {"max": 2, "interval": 10} - - async def test_allows_messages_within_limit(self): - """Cases with a total amount of role mentions within limit.""" - cases = ( - [make_msg("bob", 2)], - [make_msg("bob", 1), make_msg("alice", 1), make_msg("bob", 1)], - ) - - await self.run_allowed(cases) - - async def test_disallows_messages_beyond_limit(self): - """Cases with more than the allowed amount of role mentions.""" - cases = ( - DisallowedCase( - [make_msg("bob", 3)], - ("bob",), - 3, - ), - DisallowedCase( - [make_msg("alice", 2), make_msg("bob", 2), make_msg("alice", 2)], - ("alice",), - 4, - ), - ) - - await self.run_disallowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_message = case.recent_messages[0] - return tuple( - msg - for msg in case.recent_messages - if msg.author == last_message.author - ) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} role mentions in {self.config['interval']}s" From 8095800ae8f38928ab8c406e622ec79ea93b21c3 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 9 Dec 2021 00:30:41 +0200 Subject: [PATCH 002/132] New filtering backbone and regex filtering migration This commit provides the basis of the new filtering system: - The filtering cog consists of several filter lists loaded from the database (filtering.py). - Each filter list contains a list of filters, which are run in response to events (message posting, reaction, thread creation). Each filter list may choose to respond to different events (the subscribe method in filtering.py). - Each filter has settings (settings.py) which decide when it is going to be run (e.g it might be disabled in a specific channel), and what will happen if it triggers (e.g delete the offending message). - Not every filter has a value for every setting (the _settings_types package) . It will use the default settings specified by its filter list as a fallback. - Since each filter might have a different effect when triggered, we must check all relevant filters even if we found a triggered filter already, unlike in the old system. - Two triggered filters may specify different values for the same setting, therefore each entry has a rule for combining two different values (the __or__ method in each file in _settings_types). To avoid having to prefix each file with an underscore (or the bot will try to load it as a cog), the loading script was changed to ignore packages with names starting with an underscore. Alert sending is done via a webhook so that several embeds can be sent in the same message (will be useful for example for guild invite alerts). Filter lists and setting entries classes are loaded dynamically from their respective packages. In order to be able to test the new features, this commit also includes a migration of the regex-based filtering. --- bot/constants.py | 1 + bot/exts/filtering/README.md | 0 bot/exts/filtering/__init__.py | 0 bot/exts/filtering/_filter_context.py | 39 +++ bot/exts/filtering/_filter_lists/__init__.py | 9 + .../filtering/_filter_lists/filter_list.py | 79 +++++ bot/exts/filtering/_filter_lists/token.py | 45 +++ bot/exts/filtering/_filters/__init__.py | 0 bot/exts/filtering/_filters/filter.py | 29 ++ bot/exts/filtering/_filters/token.py | 20 ++ bot/exts/filtering/_settings.py | 180 ++++++++++++ .../filtering/_settings_types/__init__.py | 14 + .../filtering/_settings_types/bypass_roles.py | 29 ++ .../_settings_types/channel_scope.py | 45 +++ .../_settings_types/delete_messages.py | 35 +++ bot/exts/filtering/_settings_types/enabled.py | 18 ++ .../filtering/_settings_types/filter_dm.py | 18 ++ .../infraction_and_notification.py | 180 ++++++++++++ bot/exts/filtering/_settings_types/ping.py | 52 ++++ .../filtering/_settings_types/send_alert.py | 26 ++ .../_settings_types/settings_entry.py | 85 ++++++ bot/exts/filtering/_utils.py | 97 +++++++ bot/exts/filtering/filtering.py | 150 ++++++++++ bot/utils/messages.py | 9 + config-default.yml | 1 + tests/bot/exts/filtering/__init__.py | 0 tests/bot/exts/filtering/test_filters.py | 41 +++ tests/bot/exts/filtering/test_settings.py | 20 ++ .../exts/filtering/test_settings_entries.py | 272 ++++++++++++++++++ 29 files changed, 1494 insertions(+) create mode 100644 bot/exts/filtering/README.md create mode 100644 bot/exts/filtering/__init__.py create mode 100644 bot/exts/filtering/_filter_context.py create mode 100644 bot/exts/filtering/_filter_lists/__init__.py create mode 100644 bot/exts/filtering/_filter_lists/filter_list.py create mode 100644 bot/exts/filtering/_filter_lists/token.py create mode 100644 bot/exts/filtering/_filters/__init__.py create mode 100644 bot/exts/filtering/_filters/filter.py create mode 100644 bot/exts/filtering/_filters/token.py create mode 100644 bot/exts/filtering/_settings.py create mode 100644 bot/exts/filtering/_settings_types/__init__.py create mode 100644 bot/exts/filtering/_settings_types/bypass_roles.py create mode 100644 bot/exts/filtering/_settings_types/channel_scope.py create mode 100644 bot/exts/filtering/_settings_types/delete_messages.py create mode 100644 bot/exts/filtering/_settings_types/enabled.py create mode 100644 bot/exts/filtering/_settings_types/filter_dm.py create mode 100644 bot/exts/filtering/_settings_types/infraction_and_notification.py create mode 100644 bot/exts/filtering/_settings_types/ping.py create mode 100644 bot/exts/filtering/_settings_types/send_alert.py create mode 100644 bot/exts/filtering/_settings_types/settings_entry.py create mode 100644 bot/exts/filtering/_utils.py create mode 100644 bot/exts/filtering/filtering.py create mode 100644 tests/bot/exts/filtering/__init__.py create mode 100644 tests/bot/exts/filtering/test_filters.py create mode 100644 tests/bot/exts/filtering/test_settings.py create mode 100644 tests/bot/exts/filtering/test_settings_entries.py diff --git a/bot/constants.py b/bot/constants.py index c39f9d2b84..65791daa3e 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -477,6 +477,7 @@ class Webhooks(metaclass=YAMLGetter): duck_pond: int incidents: int incidents_archive: int + filters: int class Roles(metaclass=YAMLGetter): diff --git a/bot/exts/filtering/README.md b/bot/exts/filtering/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bot/exts/filtering/__init__.py b/bot/exts/filtering/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py new file mode 100644 index 0000000000..ee9e87f56e --- /dev/null +++ b/bot/exts/filtering/_filter_context.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from enum import Enum, auto +from typing import Optional, Union + +from discord import DMChannel, Embed, Message, TextChannel, Thread, User + + +class Event(Enum): + """Types of events that can trigger filtering. Note this does not have to align with gateway event types.""" + + MESSAGE = auto() + MESSAGE_EDIT = auto() + + +@dataclass +class FilterContext: + """A dataclass containing the information that should be filtered, and output information of the filtering.""" + + # Input context + event: Event # The type of event + author: User # Who triggered the event + channel: Union[TextChannel, Thread, DMChannel] # The channel involved + content: str # What actually needs filtering + message: Optional[Message] # The message involved + embeds: list = field(default_factory=list) # Any embeds involved + # Output context + dm_content: str = field(default_factory=str) # The content to DM the invoker + dm_embed: Embed = field(default_factory=Embed) # The embed to DM the invoker + send_alert: bool = field(default=True) # Whether to send an alert for the moderators + alert_content: str = field(default_factory=str) # The content of the alert + alert_embeds: list = field(default_factory=list) # Any embeds to add to the alert + action_descriptions: list = field(default_factory=list) # What actions were taken + matches: list = field(default_factory=list) # What exactly was found + + def replace(self, **changes) -> FilterContext: + """Return a new context object assigning new values to the specified fields.""" + return replace(self, **changes) diff --git a/bot/exts/filtering/_filter_lists/__init__.py b/bot/exts/filtering/_filter_lists/__init__.py new file mode 100644 index 0000000000..415e3a6bf6 --- /dev/null +++ b/bot/exts/filtering/_filter_lists/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname + +from bot.exts.filtering._filter_lists.filter_list import FilterList +from bot.exts.filtering._utils import subclasses_in_package + +filter_list_types = subclasses_in_package(dirname(__file__), f"{__name__}.", FilterList) +filter_list_types = {filter_list.name: filter_list for filter_list in filter_list_types} + +__all__ = [filter_list_types, FilterList] diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py new file mode 100644 index 0000000000..f9e304b599 --- /dev/null +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -0,0 +1,79 @@ +from abc import abstractmethod +from enum import Enum +from typing import Dict, List, Type + +from bot.exts.filtering._settings import Settings, ValidationSettings, create_settings +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._utils import FieldRequiring +from bot.log import get_logger + +log = get_logger(__name__) + + +class ListType(Enum): + DENY = 0 + ALLOW = 1 + + +class FilterList(FieldRequiring): + """Dispatches events to lists of _filters, and aggregates the responses into a single list of actions to take.""" + + # Each subclass must define a name matching the filter_list name we're expecting to receive from the database. + # Names must be unique across all filter lists. + name = FieldRequiring.MUST_SET_UNIQUE + + def __init__(self, filter_type: Type[Filter]): + self._filter_lists: dict[ListType, list[Filter]] = {} + self._defaults: dict[ListType, dict[str, Settings]] = {} + + self.filter_type = filter_type + + def add_list(self, list_data: Dict) -> None: + """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" + actions, validations = create_settings(list_data["settings"]) + list_type = ListType(list_data["list_type"]) + self._defaults[list_type] = {"actions": actions, "validations": validations} + + filters = [] + for filter_data in list_data["filters"]: + try: + filters.append(self.filter_type(filter_data, actions)) + except TypeError as e: + log.warning(e) + self._filter_lists[list_type] = filters + + @abstractmethod + def triggers_for(self, ctx: FilterContext) -> list[Filter]: + """Dispatch the given event to the list's filters, and return filters triggered.""" + + @staticmethod + def filter_list_result(ctx: FilterContext, filters: List[Filter], defaults: ValidationSettings) -> list[Filter]: + """ + Sift through the list of filters, and return only the ones which apply to the given context. + + The strategy is as follows: + 1. The default settings are evaluated on the given context. The default answer for whether the filter is + relevant in the given context is whether there aren't any validation settings which returned False. + 2. For each filter, its overrides are considered: + - If there are no overrides, then the filter is relevant if that is the default answer. + - Otherwise it is relevant if there are no failed overrides, and any failing default is overridden by a + successful override. + + If the filter is relevant in context, see if it actually triggers. + """ + passed_by_default, failed_by_default = defaults.evaluate(ctx) + default_answer = not bool(failed_by_default) + + relevant_filters = [] + for filter_ in filters: + if not filter_.validations: + if default_answer and filter_.triggered_on(ctx): + relevant_filters.append(filter_) + else: + passed, failed = filter_.validations.evaluate(ctx) + if not failed and failed_by_default < passed: + if filter_.triggered_on(ctx): + relevant_filters.append(filter_) + + return relevant_filters diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py new file mode 100644 index 0000000000..4495f44142 --- /dev/null +++ b/bot/exts/filtering/_filter_lists/token.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import re +import typing + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._filters.token import TokenFilter +from bot.exts.filtering._utils import clean_input + +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + +SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL) + + +class TokensList(FilterList): + """A list of filters, each looking for a specific token given by regex.""" + + name = "token" + + def __init__(self, filtering_cog: Filtering): + super().__init__(TokenFilter) + filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + + def triggers_for(self, ctx: FilterContext) -> list[Filter]: + """Dispatch the given event to the list's filters, and return filters triggered.""" + text = ctx.content + if SPOILER_RE.search(text): + text = self._expand_spoilers(text) + text = clean_input(text) + ctx = ctx.replace(content=text) + + return self.filter_list_result( + ctx, self._filter_lists[ListType.DENY], self._defaults[ListType.DENY]["validations"] + ) + + @staticmethod + def _expand_spoilers(text: str) -> str: + """Return a string containing all interpretations of a spoilered message.""" + split_text = SPOILER_RE.split(text) + return ''.join( + split_text[0::2] + split_text[1::2] + split_text + ) diff --git a/bot/exts/filtering/_filters/__init__.py b/bot/exts/filtering/_filters/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py new file mode 100644 index 0000000000..484e506fcf --- /dev/null +++ b/bot/exts/filtering/_filters/filter.py @@ -0,0 +1,29 @@ +from abc import ABC, abstractmethod +from typing import Dict, Optional + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings import ActionSettings, create_settings + + +class Filter(ABC): + """ + A class representing a filter. + + Each filter looks for a specific attribute within an event (such as message sent), + and defines what action should be performed if it is triggered. + """ + + def __init__(self, filter_data: Dict, action_defaults: Optional[ActionSettings] = None): + self.id = filter_data["id"] + self.content = filter_data["content"] + self.description = filter_data["description"] + self.actions, self.validations = create_settings(filter_data["settings"]) + if not self.actions: + self.actions = action_defaults + elif action_defaults: + self.actions.fallback_to(action_defaults) + self.exact = filter_data["additional_field"] + + @abstractmethod + def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" diff --git a/bot/exts/filtering/_filters/token.py b/bot/exts/filtering/_filters/token.py new file mode 100644 index 0000000000..07590c54b3 --- /dev/null +++ b/bot/exts/filtering/_filters/token.py @@ -0,0 +1,20 @@ +import re + +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._filter_context import FilterContext + + +class TokenFilter(Filter): + """A filter which looks for a specific token given by regex.""" + + def triggered_on(self, ctx: FilterContext) -> bool: + """Searches for a regex pattern within a given context.""" + pattern = self.content + + match = re.search(pattern, ctx.content, flags=re.IGNORECASE) + if match: + ctx.matches.append(match[0]) + return True + return False + + diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py new file mode 100644 index 0000000000..96e1c1f7f4 --- /dev/null +++ b/bot/exts/filtering/_settings.py @@ -0,0 +1,180 @@ +from __future__ import annotations +from abc import abstractmethod +from typing import Iterator, Mapping, Optional + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types import settings_types +from bot.exts.filtering._settings_types.settings_entry import ActionEntry, ValidationEntry +from bot.exts.filtering._utils import FieldRequiring +from bot.log import get_logger + +log = get_logger(__name__) + +_already_warned: set[str] = set() + + +def create_settings(settings_data: dict) -> tuple[Optional[ActionSettings], Optional[ValidationSettings]]: + """ + Create and return instances of the Settings subclasses from the given data + + Additionally, warn for data entries with no matching class. + """ + action_data = {} + validation_data = {} + for entry_name, entry_data in settings_data.items(): + if entry_name in settings_types["ActionEntry"]: + action_data[entry_name] = entry_data + elif entry_name in settings_types["ValidationEntry"]: + validation_data[entry_name] = entry_data + else: + log.warning( + f"A setting named {entry_name} was loaded from the database, but no matching class." + ) + _already_warned.add(entry_name) + return ActionSettings.create(action_data), ValidationSettings.create(validation_data) + + +class Settings(FieldRequiring): + """ + A collection of settings. + + For processing the settings parts in the database and evaluating them on given contexts. + + Each filter list and filter has its own settings. + + A filter doesn't have to have its own settings. For every undefined setting, it falls back to the value defined in + the filter list which contains the filter. + """ + + entry_type = FieldRequiring.MUST_SET + + _already_warned: set[str] = set() + + @abstractmethod + def __init__(self, settings_data: dict): + self._entries: dict[str, Settings.entry_type] = {} + + entry_classes = settings_types.get(self.entry_type.__name__) + for entry_name, entry_data in settings_data.items(): + try: + entry_cls = entry_classes[entry_name] + except KeyError: + if entry_name not in self._already_warned: + log.warning( + f"A setting named {entry_name} was loaded from the database, " + f"but no matching {self.entry_type.__name__} class." + ) + self._already_warned.add(entry_name) + else: + try: + new_entry = entry_cls.create(entry_data) + if new_entry: + self._entries[entry_name] = new_entry + except TypeError as e: + raise TypeError( + f"Attempted to load a {entry_name} setting, but the response is malformed: {entry_data}" + ) from e + + def __contains__(self, item) -> bool: + return item in self._entries + + def __setitem__(self, key: str, value: entry_type) -> None: + self._entries[key] = value + + def copy(self): + copy = self.__class__({}) + copy._entries = self._entries + return copy + + def items(self) -> Iterator[tuple[str, entry_type]]: + yield from self._entries.items() + + def update(self, mapping: Mapping[str, entry_type], **kwargs: entry_type) -> None: + self._entries.update(mapping, **kwargs) + + @classmethod + def create(cls, settings_data: dict) -> Optional[Settings]: + """ + Returns a Settings object from `settings_data` if it holds any value, None otherwise. + + Use this method to create Settings objects instead of the init. + The None value is significant for how a filter list iterates over its filters. + """ + settings = cls(settings_data) + # If an entry doesn't hold any values, its `create` method will return None. + # If all entries are None, then the settings object holds no values. + if not any(settings._entries.values()): + return None + + return settings + + +class ValidationSettings(Settings): + """ + A collection of validation settings. + + A filter is triggered only if all of its validation settings (e.g whether to invoke in DM) approve + (the check returns True). + """ + + entry_type = ValidationEntry + + def __init__(self, settings_data: dict): + super().__init__(settings_data) + + def evaluate(self, ctx: FilterContext) -> tuple[set[str], set[str]]: + """Evaluates for each setting whether the context is relevant to the filter.""" + passed = set() + failed = set() + + self._entries: dict[str, ValidationEntry] + for name, validation in self._entries.items(): + if validation: + if validation.triggers_on(ctx): + passed.add(name) + else: + failed.add(name) + + return passed, failed + + +class ActionSettings(Settings): + """ + A collection of action settings. + + If a filter is triggered, its action settings (e.g how to infract the user) are combined with the action settings of + other triggered filters in the same event, and action is taken according to the combined action settings. + """ + + entry_type = ActionEntry + + def __init__(self, settings_data: dict): + super().__init__(settings_data) + + def __or__(self, other: ActionSettings) -> ActionSettings: + """Combine the entries of two collections of settings into a new ActionsSettings""" + actions = {} + # A settings object doesn't necessarily have all types of entries (e.g in the case of filter overrides). + for entry in self._entries: + if entry in other._entries: + actions[entry] = self._entries[entry] | other._entries[entry] + else: + actions[entry] = self._entries[entry] + for entry in other._entries: + if entry not in actions: + actions[entry] = other._entries[entry] + + result = ActionSettings({}) + result.update(actions) + return result + + async def action(self, ctx: FilterContext) -> None: + """Execute the action of every action entry stored.""" + for entry in self._entries.values(): + await entry.action(ctx) + + def fallback_to(self, fallback: ActionSettings) -> None: + """Fill in missing entries from `fallback`.""" + for entry_name, entry_value in fallback.items(): + if entry_name not in self._entries: + self._entries[entry_name] = entry_value diff --git a/bot/exts/filtering/_settings_types/__init__.py b/bot/exts/filtering/_settings_types/__init__.py new file mode 100644 index 0000000000..620290cb27 --- /dev/null +++ b/bot/exts/filtering/_settings_types/__init__.py @@ -0,0 +1,14 @@ +from os.path import dirname + +from bot.exts.filtering._settings_types.settings_entry import ActionEntry, ValidationEntry +from bot.exts.filtering._utils import subclasses_in_package + +action_types = subclasses_in_package(dirname(__file__), f"{__name__}.", ActionEntry) +validation_types = subclasses_in_package(dirname(__file__), f"{__name__}.", ValidationEntry) + +settings_types = { + "ActionEntry": {settings_type.name: settings_type for settings_type in action_types}, + "ValidationEntry": {settings_type.name: settings_type for settings_type in validation_types} +} + +__all__ = [settings_types] diff --git a/bot/exts/filtering/_settings_types/bypass_roles.py b/bot/exts/filtering/_settings_types/bypass_roles.py new file mode 100644 index 0000000000..9665283ffd --- /dev/null +++ b/bot/exts/filtering/_settings_types/bypass_roles.py @@ -0,0 +1,29 @@ +from typing import Any + +from discord import Member + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types.settings_entry import ValidationEntry +from bot.exts.filtering._utils import ROLE_LITERALS + + +class RoleBypass(ValidationEntry): + """A setting entry which tells whether the roles the member has allow them to bypass the filter.""" + + name = "bypass_roles" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + self.roles = set() + for role in entry_data: + if role in ROLE_LITERALS: + self.roles.add(ROLE_LITERALS[role]) + elif role.isdigit(): + self.roles.add(int(role)) + # Ignore entries that can't be resolved. + + def triggers_on(self, ctx: FilterContext) -> bool: + """Return whether the filter should be triggered on this user given their roles.""" + if not isinstance(ctx.author, Member): + return True + return all(member_role.id not in self.roles for member_role in ctx.author.roles) diff --git a/bot/exts/filtering/_settings_types/channel_scope.py b/bot/exts/filtering/_settings_types/channel_scope.py new file mode 100644 index 0000000000..b17914f2ff --- /dev/null +++ b/bot/exts/filtering/_settings_types/channel_scope.py @@ -0,0 +1,45 @@ +from typing import Any + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types.settings_entry import ValidationEntry + + +class ChannelScope(ValidationEntry): + """A setting entry which tells whether the filter was invoked in a whitelisted channel or category.""" + + name = "channel_scope" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + if entry_data["disabled_channels"]: + self.disabled_channels = set(entry_data["disabled_channels"]) + else: + self.disabled_channels = set() + + if entry_data["disabled_categories"]: + self.disabled_categories = set(entry_data["disabled_categories"]) + else: + self.disabled_categories = set() + + if entry_data["enabled_channels"]: + self.enabled_channels = set(entry_data["enabled_channels"]) + else: + self.enabled_channels = set() + + def triggers_on(self, ctx: FilterContext) -> bool: + """ + Return whether the filter should be triggered in the given channel. + + The filter is invoked by default. + If the channel is explicitly enabled, it bypasses the set disabled channels and categories. + """ + channel = ctx.channel + if hasattr(channel, "parent"): + channel = channel.parent + return ( + channel.id in self.enabled_channels + or ( + channel.id not in self.disabled_channels + and (not channel.category or channel.category.id not in self.disabled_categories) + ) + ) diff --git a/bot/exts/filtering/_settings_types/delete_messages.py b/bot/exts/filtering/_settings_types/delete_messages.py new file mode 100644 index 0000000000..b0a018433d --- /dev/null +++ b/bot/exts/filtering/_settings_types/delete_messages.py @@ -0,0 +1,35 @@ +from contextlib import suppress +from typing import Any + +from discord.errors import NotFound + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._settings_types.settings_entry import ActionEntry + + +class DeleteMessages(ActionEntry): + """A setting entry which tells whether to delete the offending message(s).""" + + name = "delete_messages" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + self.delete: bool = entry_data + + async def action(self, ctx: FilterContext) -> None: + """Delete the context message(s).""" + if not self.delete or ctx.event not in (Event.MESSAGE, Event.MESSAGE_EDIT): + return + + with suppress(NotFound): + if ctx.message.guild: + await ctx.message.delete() + ctx.action_descriptions.append("deleted") + + def __or__(self, other: ActionEntry): + """Combines two actions of the same type. Each type of action is executed once per filter.""" + if not isinstance(other, DeleteMessages): + return NotImplemented + + return DeleteMessages(self.delete or other.delete) + diff --git a/bot/exts/filtering/_settings_types/enabled.py b/bot/exts/filtering/_settings_types/enabled.py new file mode 100644 index 0000000000..553dccc9cf --- /dev/null +++ b/bot/exts/filtering/_settings_types/enabled.py @@ -0,0 +1,18 @@ +from typing import Any + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types.settings_entry import ValidationEntry + + +class Enabled(ValidationEntry): + """A setting entry which tells whether the filter is enabled.""" + + name = "enabled" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + self.enabled = entry_data + + def triggers_on(self, ctx: FilterContext) -> bool: + """Return whether the filter is enabled.""" + return self.enabled diff --git a/bot/exts/filtering/_settings_types/filter_dm.py b/bot/exts/filtering/_settings_types/filter_dm.py new file mode 100644 index 0000000000..54f19e4d17 --- /dev/null +++ b/bot/exts/filtering/_settings_types/filter_dm.py @@ -0,0 +1,18 @@ +from typing import Any + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types.settings_entry import ValidationEntry + + +class FilterDM(ValidationEntry): + """A setting entry which tells whether to apply the filter to DMs.""" + + name = "filter_dm" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + self.apply_in_dm = entry_data + + def triggers_on(self, ctx: FilterContext) -> bool: + """Return whether the filter should be triggered even if it was triggered in DMs.""" + return hasattr(ctx.channel, "guild") or self.apply_in_dm diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py new file mode 100644 index 0000000000..263fd851ca --- /dev/null +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -0,0 +1,180 @@ +from collections import namedtuple +from datetime import timedelta +from enum import Enum, auto +from typing import Any, Optional + +import arrow +from discord import Colour +from discord.errors import Forbidden + +import bot +from bot.constants import Channels, Guild +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types.settings_entry import ActionEntry + + +class Infraction(Enum): + """An enumeration of infraction types. The lower the value, the higher it is on the hierarchy.""" + + BAN = auto() + KICK = auto() + MUTE = auto() + VOICE_BAN = auto() + SUPERSTAR = auto() + WARNING = auto() + WATCH = auto() + NOTE = auto() + NONE = auto() # Allows making operations on an entry with no infraction without checking for None. + + def __bool__(self) -> bool: + """ + Make the NONE value false-y. + + This is useful for Settings.create to evaluate whether the entry contains anything. + """ + return self != Infraction.NONE + + +superstar = namedtuple("superstar", ["reason", "duration"]) + + +class InfractionAndNotification(ActionEntry): + """ + A setting entry which specifies what infraction to issue and the notification to DM the user. + + Since a DM cannot be sent when a user is banned or kicked, these two functions need to be grouped together. + """ + + name = "infraction_and_notification" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + + if entry_data["infraction_type"]: + self.infraction_type = entry_data["infraction_type"] + if isinstance(self.infraction_type, str): + self.infraction_type = Infraction[self.infraction_type.replace(" ", "_").upper()] + self.infraction_reason = entry_data["infraction_reason"] + if entry_data["infraction_duration"] is not None: + self.infraction_duration = float(entry_data["infraction_duration"]) + else: + self.infraction_duration = None + else: + self.infraction_type = Infraction.NONE + self.infraction_reason = None + self.infraction_duration = 0 + + self.dm_content = entry_data["dm_content"] + self.dm_embed = entry_data["dm_embed"] + + self._superstar = entry_data.get("superstar", None) + + async def action(self, ctx: FilterContext) -> None: + """Send the notification to the user, and apply any specified infractions.""" + # If there is no infraction to apply, any DM contents already provided in the context take precedence. + if self.infraction_type == Infraction.NONE and (ctx.dm_content or ctx.dm_embed): + dm_content = ctx.dm_content + dm_embed = ctx.dm_embed.description + else: + dm_content = self.dm_content + dm_embed = self.dm_embed + + if dm_content or dm_embed: + dm_content = f"Hey {ctx.author.mention}!\n{dm_content}" + ctx.dm_embed.description = dm_embed + if not ctx.dm_embed.colour: + ctx.dm_embed.colour = Colour.og_blurple() + + try: + await ctx.author.send(dm_content, embed=ctx.dm_embed) + except Forbidden: + await ctx.channel.send(ctx.dm_content, embed=ctx.dm_embed) + ctx.action_descriptions.append("notified") + + msg_ctx = await bot.instance.get_context(ctx.message) + msg_ctx.guild = bot.instance.get_guild(Guild.id) + msg_ctx.author = ctx.author + msg_ctx.channel = ctx.channel + if self._superstar: + msg_ctx.command = bot.instance.get_command("superstarify") + await msg_ctx.invoke( + msg_ctx.command, + ctx.author, + arrow.utcnow() + timedelta(seconds=self._superstar.duration) + if self._superstar.duration is not None else None, + reason=self._superstar.reason + ) + ctx.action_descriptions.append("superstar") + + if self.infraction_type != Infraction.NONE: + if self.infraction_type == Infraction.BAN or not hasattr(ctx.channel, "guild"): + msg_ctx.channel = bot.instance.get_channel(Channels.mod_alerts) + msg_ctx.command = bot.instance.get_command(self.infraction_type.name) + await msg_ctx.invoke( + msg_ctx.command, + ctx.author, + arrow.utcnow() + timedelta(seconds=self.infraction_duration) + if self.infraction_duration is not None else None, + reason=self.infraction_reason + ) + ctx.action_descriptions.append(self.infraction_type.name.lower()) + + def __or__(self, other: ActionEntry): + """ + Combines two actions of the same type. Each type of action is executed once per filter. + + If the infractions are different, take the data of the one higher up the hierarchy. + + A special case is made for superstar infractions. Even if we decide to auto-mute a user, if they have a + particularly problematic username we will still want to superstarify them. + + This is a "best attempt" implementation. Trying to account for any type of combination would create an + extremely complex ruleset. For example, we could special-case watches as well. + + There is no clear way to properly combine several notification messages, especially when it's in two parts. + To avoid bombarding the user with several notifications, the message with the more significant infraction + is used. + """ + if not isinstance(other, InfractionAndNotification): + return NotImplemented + + # Lower number -> higher in the hierarchy + if self.infraction_type.value < other.infraction_type.value and other.infraction_type != Infraction.SUPERSTAR: + result = InfractionAndNotification(self.to_dict()) + result._superstar = self._merge_superstars(self._superstar, other._superstar) + return result + elif self.infraction_type.value > other.infraction_type.value and self.infraction_type != Infraction.SUPERSTAR: + result = InfractionAndNotification(other.to_dict()) + result._superstar = self._merge_superstars(self._superstar, other._superstar) + return result + + if self.infraction_type == other.infraction_type: + if self.infraction_duration is None or ( + other.infraction_duration is not None and self.infraction_duration > other.infraction_duration + ): + result = InfractionAndNotification(self.to_dict()) + else: + result = InfractionAndNotification(other.to_dict()) + result._superstar = self._merge_superstars(self._superstar, other._superstar) + return result + + # At this stage the infraction types are different, and the lower one is a superstar. + if self.infraction_type.value < other.infraction_type.value: + result = InfractionAndNotification(self.to_dict()) + result._superstar = superstar(other.infraction_reason, other.infraction_duration) + else: + result = InfractionAndNotification(other.to_dict()) + result._superstar = superstar(self.infraction_reason, self.infraction_duration) + return result + + @staticmethod + def _merge_superstars(superstar1: Optional[superstar], superstar2: Optional[superstar]) -> Optional[superstar]: + """Take the superstar with the greater duration.""" + if not superstar1: + return superstar2 + if not superstar2: + return superstar1 + + if superstar1.duration is None or superstar1.duration > superstar2.duration: + return superstar1 + return superstar2 diff --git a/bot/exts/filtering/_settings_types/ping.py b/bot/exts/filtering/_settings_types/ping.py new file mode 100644 index 0000000000..857e4a7e8f --- /dev/null +++ b/bot/exts/filtering/_settings_types/ping.py @@ -0,0 +1,52 @@ +from functools import cache +from typing import Any + +from discord import Guild + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types.settings_entry import ActionEntry +from bot.exts.filtering._utils import ROLE_LITERALS + + +class Ping(ActionEntry): + """A setting entry which adds the appropriate pings to the alert.""" + + name = "mentions" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + self.guild_mentions = set(entry_data["guild_pings"]) + self.dm_mentions = set(entry_data["dm_pings"]) + + async def action(self, ctx: FilterContext) -> None: + """Add the stored pings to the alert message content.""" + mentions = self.guild_mentions if ctx.channel.guild else self.dm_mentions + new_content = " ".join([self._resolve_mention(mention, ctx.channel.guild) for mention in mentions]) + ctx.alert_content = f"{new_content} {ctx.alert_content}" + + def __or__(self, other: ActionEntry): + """Combines two actions of the same type. Each type of action is executed once per filter.""" + if not isinstance(other, Ping): + return NotImplemented + + return Ping({ + "ping_type": self.guild_mentions | other.guild_mentions, + "dm_ping_type": self.dm_mentions | other.dm_mentions + }) + + @staticmethod + @cache + def _resolve_mention(mention: str, guild: Guild) -> str: + """Return the appropriate formatting for the formatting, be it a literal, a user ID, or a role ID.""" + if mention in ("here", "everyone"): + return f"@{mention}" + if mention in ROLE_LITERALS: + return f"<@&{ROLE_LITERALS[mention]}>" + if not mention.isdigit(): + return mention + + mention = int(mention) + if any(mention == role.id for role in guild.roles): + return f"<@&{mention}>" + else: + return f"<@{mention}>" diff --git a/bot/exts/filtering/_settings_types/send_alert.py b/bot/exts/filtering/_settings_types/send_alert.py new file mode 100644 index 0000000000..e332494eb5 --- /dev/null +++ b/bot/exts/filtering/_settings_types/send_alert.py @@ -0,0 +1,26 @@ +from typing import Any + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._settings_types.settings_entry import ActionEntry + + +class SendAlert(ActionEntry): + """A setting entry which tells whether to send an alert message.""" + + name = "send_alert" + + def __init__(self, entry_data: Any): + super().__init__(entry_data) + self.send_alert: bool = entry_data + + async def action(self, ctx: FilterContext) -> None: + """Add the stored pings to the alert message content.""" + ctx.send_alert = self.send_alert + + def __or__(self, other: ActionEntry): + """Combines two actions of the same type. Each type of action is executed once per filter.""" + if not isinstance(other, SendAlert): + return NotImplemented + + return SendAlert(self.send_alert or other.send_alert) + diff --git a/bot/exts/filtering/_settings_types/settings_entry.py b/bot/exts/filtering/_settings_types/settings_entry.py new file mode 100644 index 0000000000..b0d54fac37 --- /dev/null +++ b/bot/exts/filtering/_settings_types/settings_entry.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from abc import abstractmethod +from typing import Any, Optional + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._utils import FieldRequiring + + +class SettingsEntry(FieldRequiring): + """ + A basic entry in the settings field appearing in every filter list and filter. + + For a filter list, this is the default setting for it. For a filter, it's an override of the default entry. + """ + + # Each subclass must define a name matching the entry name we're expecting to receive from the database. + # Names must be unique across all filter lists. + name = FieldRequiring.MUST_SET_UNIQUE + + @abstractmethod + def __init__(self, entry_data: Any): + super().__init__() + self._dict = {} + + def __setattr__(self, key: str, value: Any) -> None: + super().__setattr__(key, value) + if key == "_dict": + return + self._dict[key] = value + + def __eq__(self, other: SettingsEntry) -> bool: + if not isinstance(other, SettingsEntry): + return NotImplemented + return self._dict == other._dict + + def to_dict(self) -> dict[str, Any]: + """Return a dictionary representation of the entry.""" + return self._dict.copy() + + def copy(self) -> SettingsEntry: + """Return a new entry object with the same parameters.""" + return self.__class__(self.to_dict()) + + @classmethod + def create(cls, entry_data: Optional[dict[str, Any]]) -> Optional[SettingsEntry]: + """ + Returns a SettingsEntry object from `entry_data` if it holds any value, None otherwise. + + Use this method to create SettingsEntry objects instead of the init. + The None value is significant for how a filter list iterates over its filters. + """ + if entry_data is None: + return None + if hasattr(entry_data, "values") and not any(value for value in entry_data.values()): + return None + + return cls(entry_data) + + +class ValidationEntry(SettingsEntry): + """A setting entry to validate whether the filter should be triggered in the given context.""" + + @abstractmethod + def triggers_on(self, ctx: FilterContext) -> bool: + """Return whether the filter should be triggered with this setting in the given context.""" + ... + + +class ActionEntry(SettingsEntry): + """A setting entry defining what the bot should do if the filter it belongs to is triggered.""" + + @abstractmethod + async def action(self, ctx: FilterContext) -> None: + """Execute an action that should be taken when the filter this setting belongs to is triggered.""" + ... + + @abstractmethod + def __or__(self, other: ActionEntry): + """ + Combine two actions of the same type. Each type of action is executed once per filter. + + The following condition must hold: if self == other, then self | other == self. + """ + ... diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py new file mode 100644 index 0000000000..a769001f61 --- /dev/null +++ b/bot/exts/filtering/_utils.py @@ -0,0 +1,97 @@ +import importlib +import importlib.util +import inspect +import pkgutil +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import Set + +import regex + +from bot.constants import Roles + +ROLE_LITERALS = { + "admins": Roles.admins, + "onduty": Roles.moderators, + "staff": Roles.helpers +} + +VARIATION_SELECTORS = r"\uFE00-\uFE0F\U000E0100-\U000E01EF" +INVISIBLE_RE = regex.compile(rf"[{VARIATION_SELECTORS}\p{{UNASSIGNED}}\p{{FORMAT}}\p{{CONTROL}}--\s]", regex.V1) +ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIATION_SELECTORS}]]", regex.V1) + + +def subclasses_in_package(package: str, prefix: str, parent: type) -> Set[type]: + """Return all the subclasses of class `parent`, found in the top-level of `package`, given by absolute path.""" + subclasses = set() + + # Find all modules in the package. + for module_info in pkgutil.iter_modules([package], prefix): + if not module_info.ispkg: + module = importlib.import_module(module_info.name) + # Find all classes in each module... + for _, class_ in inspect.getmembers(module, inspect.isclass): + # That are a subclass of the given class. + if parent in class_.__bases__: + subclasses.add(class_) + + return subclasses + + +def clean_input(string: str) -> str: + """Remove zalgo and invisible characters from `string`.""" + # For future consideration: remove characters in the Mc, Sk, and Lm categories too. + # Can be normalised with form C to merge char + combining char into a single char to avoid + # removing legit diacritics, but this would open up a way to bypass _filters. + no_zalgo = ZALGO_RE.sub("", string) + return INVISIBLE_RE.sub("", no_zalgo) + + +class FieldRequiring(ABC): + """A mixin class that can force its concrete subclasses to set a value for specific class attributes.""" + + # Sentinel value that mustn't remain in a concrete subclass. + MUST_SET = object() + + # Sentinel value that mustn't remain in a concrete subclass. + # Overriding value must be unique in the subclasses of the abstract class in which the attribute was set. + MUST_SET_UNIQUE = object() + + # A mapping of the attributes which must be unique, and their unique values, per FieldRequiring subclass. + __unique_attributes: defaultdict[type, dict[str, set]] = defaultdict(dict) + + @abstractmethod + def __init__(self): + ... + + def __init_subclass__(cls, **kwargs): + # If a new attribute with the value MUST_SET_UNIQUE was defined in an abstract class, record it. + if inspect.isabstract(cls): + for attribute in dir(cls): + if getattr(cls, attribute, None) is FieldRequiring.MUST_SET_UNIQUE: + for parent in cls.__mro__[1:-1]: # The first element is the class itself, last element is object. + if hasattr(parent, attribute): # The attribute was inherited. + break + else: + # A new attribute with the value MUST_SET_UNIQUE. + FieldRequiring.__unique_attributes[cls][attribute] = set() + return + + for attribute in dir(cls): + if attribute.startswith("__") or attribute in ("MUST_SET", "MUST_SET_UNIQUE"): + continue + value = getattr(cls, attribute) + if value is FieldRequiring.MUST_SET: + raise ValueError(f"You must set attribute {attribute!r} when creating {cls!r}") + elif value is FieldRequiring.MUST_SET_UNIQUE: + raise ValueError(f"You must set a unique value to attribute {attribute!r} when creating {cls!r}") + else: + # Check if the value needs to be unique. + for parent in cls.__mro__[1:-1]: + # Find the parent class the attribute was first defined in. + if attribute in FieldRequiring.__unique_attributes[parent]: + if value in FieldRequiring.__unique_attributes[parent][attribute]: + raise ValueError(f"Value of {attribute!r} in {cls!r} is not unique for parent {parent!r}.") + else: + # Add to the set of unique values for that field. + FieldRequiring.__unique_attributes[parent][attribute].add(value) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py new file mode 100644 index 0000000000..c74b856988 --- /dev/null +++ b/bot/exts/filtering/filtering.py @@ -0,0 +1,150 @@ +import operator +from collections import defaultdict +from functools import reduce +from typing import Optional + +from discord import Embed, HTTPException, Message +from discord.ext.commands import Cog +from discord.utils import escape_markdown + +from bot.bot import Bot +from bot.constants import Colours, Webhooks +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_lists import FilterList, filter_list_types +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._settings import ActionSettings +from bot.log import get_logger +from bot.utils.messages import format_channel, format_user + +log = get_logger(__name__) + + +class Filtering(Cog): + """Filtering and alerting for content posted on the server.""" + + def __init__(self, bot: Bot): + self.bot = bot + self.filter_lists: dict[str, FilterList] = {} + self._subscriptions: defaultdict[Event, list[FilterList]] = defaultdict(list) + self.webhook = None + + async def cog_load(self) -> None: + """ + Fetch the filter data from the API, parse it, and load it to the appropriate data structures. + + Additionally, fetch the alerting webhook. + """ + await self.bot.wait_until_guild_available() + already_warned = set() + + raw_filter_lists = await self.bot.api_client.get("bot/filter/filter_lists") + for raw_filter_list in raw_filter_lists: + list_name = raw_filter_list["name"] + if list_name not in self.filter_lists: + if list_name not in filter_list_types: + if list_name not in already_warned: + log.warning( + f"A filter list named {list_name} was loaded from the database, but no matching class." + ) + already_warned.add(list_name) + continue + self.filter_lists[list_name] = filter_list_types[list_name](self) + self.filter_lists[list_name].add_list(raw_filter_list) + + try: + self.webhook = await self.bot.fetch_webhook(Webhooks.filters) + except HTTPException: + log.error(f"Failed to fetch incidents webhook with id `{Webhooks.incidents}`.") + + def subscribe(self, filter_list: FilterList, *events: Event) -> None: + """ + Subscribe a filter list to the given events. + + The filter list is added to a list for each event. When the event is triggered, the filter context will be + dispatched to the subscribed filter lists. + + While it's possible to just make each filter list check the context's event, these are only the events a filter + list expects to receive from the filtering cog, there isn't an actual limitation on the kinds of events a filter + list can handle as long as the filter context is built properly. If for whatever reason we want to invoke a + filter list outside of the usual procedure with the filtering cog, it will be more problematic if the events are + hard-coded into each filter list. + """ + for event in events: + if filter_list not in self._subscriptions[event]: + self._subscriptions[event].append(filter_list) + + async def _resolve_action( + self, ctx: FilterContext + ) -> tuple[dict[FilterList, list[Filter]], Optional[ActionSettings]]: + """Get the filters triggered per list, and resolve from them the action that needs to be taken for the event.""" + triggered = {} + for filter_list in self._subscriptions[ctx.event]: + triggered[filter_list] = filter_list.triggers_for(ctx) + + result_actions = None + if triggered: + result_actions = reduce( + operator.or_, (filter_.actions for filters in triggered.values() for filter_ in filters) + ) + + return triggered, result_actions + + @Cog.listener() + async def on_message(self, msg: Message) -> None: + """Filter the contents of a sent message.""" + if msg.author.bot: + return + + ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) + + triggered, result_actions = await self._resolve_action(ctx) + if result_actions: + await result_actions.action(ctx) + if ctx.send_alert: + await self._send_alert(ctx, triggered) + + async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, list[Filter]]) -> None: + """Build an alert message from the filter context, and send it via the alert webhook.""" + if not self.webhook: + return + + name = f"{ctx.event.name.replace('_', ' ').title()} Filter" + + embed = Embed(color=Colours.soft_orange) + embed.set_thumbnail(url=ctx.author.display_avatar.url) + triggered_by = f"**Triggered by:** {format_user(ctx.author)}" + if ctx.channel.guild: + triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}" + else: + triggered_in = "**DM**" + if len(triggered_filters) == 1 and len(list(triggered_filters.values())[0]) == 1: + filter_list, (filter_,) = next(iter(triggered_filters.items())) + filters = f"**{filter_list.name.title()} Filter:** #{filter_.id} (`{filter_.content}`)" + if filter_.description: + filters += f" - {filter_.description}" + else: + filters = [] + for filter_list, list_filters in triggered_filters.items(): + filters.append( + (f"**{filter_list.name.title()} Filters:** " + ", ".join(f"#{filter_.id} (`{filter_.content}`)" for filter_ in list_filters)) + ) + filters = "\n".join(filters) + + matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) + actions = "**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") + content = f"**[Original Content]({ctx.message.jump_url})**: {escape_markdown(ctx.content)}" + + embed_content = "\n".join( + part for part in (triggered_by, triggered_in, filters, matches, actions, content) if part + ) + if len(embed_content) > 4000: + embed_content = embed_content[:4000] + " [...]" + embed.description = embed_content + + await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds]) + + +async def setup(bot: Bot) -> None: + """Load the Filtering cog.""" + await bot.add_cog(Filtering(bot)) diff --git a/bot/utils/messages.py b/bot/utils/messages.py index a5ed84351d..63929cd0b8 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -238,3 +238,12 @@ async def send_denial(ctx: Context, reason: str) -> discord.Message: def format_user(user: discord.abc.User) -> str: """Return a string for `user` which has their mention and ID.""" return f"{user.mention} (`{user.id}`)" + + +def format_channel(channel: discord.abc.Messageable) -> str: + """Return a string for `channel` with its mention, ID, and the parent channel if it is a thread.""" + formatted = f"{channel.mention} ({channel.category}/#{channel}" + if hasattr(channel, "parent"): + formatted += f"/{channel.parent}" + formatted += ")" + return formatted diff --git a/config-default.yml b/config-default.yml index 91945e2b84..1815b8ed73 100644 --- a/config-default.yml +++ b/config-default.yml @@ -317,6 +317,7 @@ guild: incidents: 816650601844572212 incidents_archive: 720671599790915702 python_news: &PYNEWS_WEBHOOK 704381182279942324 + filters: 926442964463521843 filter: diff --git a/tests/bot/exts/filtering/__init__.py b/tests/bot/exts/filtering/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/bot/exts/filtering/test_filters.py b/tests/bot/exts/filtering/test_filters.py new file mode 100644 index 0000000000..214637b525 --- /dev/null +++ b/tests/bot/exts/filtering/test_filters.py @@ -0,0 +1,41 @@ +import unittest + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.token import TokenFilter +from tests.helpers import MockMember, MockMessage, MockTextChannel + + +class FilterTests(unittest.TestCase): + """Test functionality of the token filter.""" + + def setUp(self) -> None: + member = MockMember(id=123) + channel = MockTextChannel(id=345) + message = MockMessage(author=member, channel=channel) + self.ctx = FilterContext(Event.MESSAGE, member, channel, "", message) + + def test_token_filter_triggers(self): + """The filter should evaluate to True only if its token is found in the context content.""" + test_cases = ( + (r"hi", "oh hi there", True), + (r"hi", "goodbye", False), + (r"bla\d{2,4}", "bla18", True), + (r"bla\d{2,4}", "bla1", False) + ) + + for pattern, content, expected in test_cases: + with self.subTest( + pattern=pattern, + content=content, + expected=expected, + ): + filter_ = TokenFilter({ + "id": 1, + "content": pattern, + "description": None, + "settings": {}, + "additional_field": "{}" # noqa: P103 + }) + self.ctx.content = content + result = filter_.triggered_on(self.ctx) + self.assertEqual(result, expected) diff --git a/tests/bot/exts/filtering/test_settings.py b/tests/bot/exts/filtering/test_settings.py new file mode 100644 index 0000000000..ac21a5d477 --- /dev/null +++ b/tests/bot/exts/filtering/test_settings.py @@ -0,0 +1,20 @@ +import unittest + +import bot.exts.filtering._settings +from bot.exts.filtering._settings import create_settings + + +class FilterTests(unittest.TestCase): + """Test functionality of the Settings class and its subclasses.""" + + def test_create_settings_returns_none_for_empty_data(self): + """`create_settings` should return a tuple of two Nones when passed an empty dict.""" + result = create_settings({}) + + self.assertEquals(result, (None, None)) + + def test_unrecognized_entry_makes_a_warning(self): + """When an unrecognized entry name is passed to `create_settings`, it should be added to `_already_warned`.""" + create_settings({"abcd": {}}) + + self.assertIn("abcd", bot.exts.filtering._settings._already_warned) diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py new file mode 100644 index 0000000000..4db6438abc --- /dev/null +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -0,0 +1,272 @@ +import unittest + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._settings_types.bypass_roles import RoleBypass +from bot.exts.filtering._settings_types.channel_scope import ChannelScope +from bot.exts.filtering._settings_types.filter_dm import FilterDM +from bot.exts.filtering._settings_types.infraction_and_notification import ( + Infraction, InfractionAndNotification, superstar +) +from tests.helpers import MockCategoryChannel, MockDMChannel, MockMember, MockMessage, MockRole, MockTextChannel + + +class FilterTests(unittest.TestCase): + """Test functionality of the Settings class and its subclasses.""" + + def setUp(self) -> None: + member = MockMember(id=123) + channel = MockTextChannel(id=345) + message = MockMessage(author=member, channel=channel) + self.ctx = FilterContext(Event.MESSAGE, member, channel, "", message) + + def test_role_bypass_is_off_for_user_without_roles(self): + """The role bypass should trigger when a user has no roles.""" + member = MockMember() + self.ctx.author = member + bypass_entry = RoleBypass(["123"]) + + result = bypass_entry.triggers_on(self.ctx) + + self.assertTrue(result) + + def test_role_bypass_is_on_for_a_user_with_the_right_role(self): + """The role bypass should not trigger when the user has one of its roles.""" + cases = ( + ([123], ["123"]), + ([123, 234], ["123"]), + ([123], ["123", "234"]), + ([123, 234], ["123", "234"]) + ) + + for user_role_ids, bypasses in cases: + with self.subTest(user_role_ids=user_role_ids, bypasses=bypasses): + user_roles = [MockRole(id=role_id) for role_id in user_role_ids] + member = MockMember(roles=user_roles) + self.ctx.author = member + bypass_entry = RoleBypass(bypasses) + + result = bypass_entry.triggers_on(self.ctx) + + self.assertFalse(result) + + def test_context_doesnt_trigger_for_empty_channel_scope(self): + """A filter is enabled for all channels by default.""" + channel = MockTextChannel() + scope = ChannelScope({"disabled_channels": None, "disabled_categories": None, "enabled_channels": None}) + self.ctx.channel = channel + + result = scope.triggers_on(self.ctx) + + self.assertTrue(result) + + def test_context_doesnt_trigger_for_disabled_channel(self): + """A filter shouldn't trigger if it's been disabled in the channel.""" + channel = MockTextChannel(id=123) + scope = ChannelScope({"disabled_channels": [123], "disabled_categories": None, "enabled_channels": None}) + self.ctx.channel = channel + + result = scope.triggers_on(self.ctx) + + self.assertFalse(result) + + def test_context_doesnt_trigger_in_disabled_category(self): + """A filter shouldn't trigger if it's been disabled in the category.""" + channel = MockTextChannel() + scope = ChannelScope({ + "disabled_channels": None, "disabled_categories": [channel.category.id], "enabled_channels": None + }) + self.ctx.channel = channel + + result = scope.triggers_on(self.ctx) + + self.assertFalse(result) + + def test_context_triggers_in_enabled_channel_in_disabled_category(self): + """A filter should trigger in an enabled channel even if it's been disabled in the category.""" + channel = MockTextChannel(id=123, category=MockCategoryChannel(id=234)) + scope = ChannelScope({"disabled_channels": None, "disabled_categories": [234], "enabled_channels": [123]}) + self.ctx.channel = channel + + result = scope.triggers_on(self.ctx) + + self.assertTrue(result) + + def test_filtering_dms_when_necessary(self): + """A filter correctly ignores or triggers in a channel depending on the value of FilterDM.""" + cases = ( + (True, MockDMChannel(), True), + (False, MockDMChannel(), False), + (True, MockTextChannel(), True), + (False, MockTextChannel(), True) + ) + + for apply_in_dms, channel, expected in cases: + with self.subTest(apply_in_dms=apply_in_dms, channel=channel): + filter_dms = FilterDM(apply_in_dms) + self.ctx.channel = channel + + result = filter_dms.triggers_on(self.ctx) + + self.assertEqual(expected, result) + + def test_infraction_merge_of_same_infraction_type(self): + """When both infractions are of the same type, the one with the longer duration wins.""" + infraction1 = InfractionAndNotification({ + "infraction_type": "mute", + "infraction_reason": "hi", + "infraction_duration": 10, + "dm_content": "how", + "dm_embed": "what is" + }) + infraction2 = InfractionAndNotification({ + "infraction_type": "mute", + "infraction_reason": "there", + "infraction_duration": 20, + "dm_content": "are you", + "dm_embed": "your name" + }) + + result = infraction1 | infraction2 + + self.assertDictEqual( + result.to_dict(), + { + "infraction_type": Infraction.MUTE, + "infraction_reason": "there", + "infraction_duration": 20.0, + "dm_content": "are you", + "dm_embed": "your name", + "_superstar": None + } + ) + + def test_infraction_merge_of_different_infraction_types(self): + """If there are two different infraction types, the one higher up the hierarchy should be picked.""" + infraction1 = InfractionAndNotification({ + "infraction_type": "mute", + "infraction_reason": "hi", + "infraction_duration": 20, + "dm_content": "", + "dm_embed": "" + }) + infraction2 = InfractionAndNotification({ + "infraction_type": "ban", + "infraction_reason": "", + "infraction_duration": 10, + "dm_content": "there", + "dm_embed": "" + }) + + result = infraction1 | infraction2 + + self.assertDictEqual( + result.to_dict(), + { + "infraction_type": Infraction.BAN, + "infraction_reason": "", + "infraction_duration": 10.0, + "dm_content": "there", + "dm_embed": "", + "_superstar": None + } + ) + + def test_infraction_merge_with_a_superstar(self): + """If there is a superstar infraction, it should be added to a separate field.""" + infraction1 = InfractionAndNotification({ + "infraction_type": "mute", + "infraction_reason": "hi", + "infraction_duration": 20, + "dm_content": "there", + "dm_embed": "" + }) + infraction2 = InfractionAndNotification({ + "infraction_type": "superstar", + "infraction_reason": "hello", + "infraction_duration": 10, + "dm_content": "you", + "dm_embed": "" + }) + + result = infraction1 | infraction2 + + self.assertDictEqual( + result.to_dict(), + { + "infraction_type": Infraction.MUTE, + "infraction_reason": "hi", + "infraction_duration": 20.0, + "dm_content": "there", + "dm_embed": "", + "_superstar": superstar("hello", 10.0) + } + ) + + def test_merge_two_superstar_infractions(self): + """When two superstar infractions are merged, the infraction type remains a superstar.""" + infraction1 = InfractionAndNotification({ + "infraction_type": "superstar", + "infraction_reason": "hi", + "infraction_duration": 20, + "dm_content": "", + "dm_embed": "" + }) + infraction2 = InfractionAndNotification({ + "infraction_type": "superstar", + "infraction_reason": "", + "infraction_duration": 10, + "dm_content": "there", + "dm_embed": "" + }) + + result = infraction1 | infraction2 + + self.assertDictEqual( + result.to_dict(), + { + "infraction_type": Infraction.SUPERSTAR, + "infraction_reason": "hi", + "infraction_duration": 20.0, + "dm_content": "", + "dm_embed": "", + "_superstar": None + } + ) + + def test_merge_a_voiceban_and_a_superstar_with_another_superstar(self): + """An infraction with a superstar merged with a superstar should combine under `_superstar`.""" + infraction1 = InfractionAndNotification({ + "infraction_type": "voice ban", + "infraction_reason": "hi", + "infraction_duration": 20, + "dm_content": "hello", + "dm_embed": "" + }) + infraction2 = InfractionAndNotification({ + "infraction_type": "superstar", + "infraction_reason": "bla", + "infraction_duration": 10, + "dm_content": "there", + "dm_embed": "" + }) + infraction3 = InfractionAndNotification({ + "infraction_type": "superstar", + "infraction_reason": "blabla", + "infraction_duration": 20, + "dm_content": "there", + "dm_embed": "" + }) + + result = infraction1 | infraction2 | infraction3 + + self.assertDictEqual( + result.to_dict(), + { + "infraction_type": Infraction.VOICE_BAN, + "infraction_reason": "hi", + "infraction_duration": 20, + "dm_content": "hello", + "dm_embed": "", + "_superstar": superstar("blabla", 20) + } + ) From 7ec96678663daed8c9aa81c2e2b65083c432e395 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 21 Feb 2022 02:30:44 +0200 Subject: [PATCH 003/132] Add listing commands For legacy purposes, separate command groups were re-added for blacklists and whitelists. There's a new command group for filters. Not specifying a list type for the `filter list` command will cause the bot to try to infer whether there's only one kind of list, for example `!filter list tokens` will pull up the blacklist since there's no whitelist. If a required field is missing, the user will be prompted to complete it from a selection. Some level of flexibility was added for the list type in the `filter list` command. For example, a list type can be "DENY", but it will also match "blacklist", "denylist", "deny", "denied", "blacklisted" etc. --- bot/exts/filtering/_filter_lists/__init__.py | 4 +- .../filtering/_filter_lists/filter_list.py | 34 +++- bot/exts/filtering/_filter_lists/token.py | 2 +- bot/exts/filtering/_filters/filter.py | 7 + bot/exts/filtering/_ui.py | 48 ++++++ bot/exts/filtering/_utils.py | 11 ++ bot/exts/filtering/filtering.py | 155 +++++++++++++++--- 7 files changed, 232 insertions(+), 29 deletions(-) create mode 100644 bot/exts/filtering/_ui.py diff --git a/bot/exts/filtering/_filter_lists/__init__.py b/bot/exts/filtering/_filter_lists/__init__.py index 415e3a6bf6..1273e5588a 100644 --- a/bot/exts/filtering/_filter_lists/__init__.py +++ b/bot/exts/filtering/_filter_lists/__init__.py @@ -1,9 +1,9 @@ from os.path import dirname -from bot.exts.filtering._filter_lists.filter_list import FilterList +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType, ListTypeConverter from bot.exts.filtering._utils import subclasses_in_package filter_list_types = subclasses_in_package(dirname(__file__), f"{__name__}.", FilterList) filter_list_types = {filter_list.name: filter_list for filter_list in filter_list_types} -__all__ = [filter_list_types, FilterList] +__all__ = [filter_list_types, FilterList, ListType, ListTypeConverter] diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index f9e304b599..1060f11dba 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -2,20 +2,40 @@ from enum import Enum from typing import Dict, List, Type -from bot.exts.filtering._settings import Settings, ValidationSettings, create_settings -from bot.exts.filtering._filters.filter import Filter +from discord.ext.commands import BadArgument, Context, Converter + from bot.exts.filtering._filter_context import FilterContext -from bot.exts.filtering._utils import FieldRequiring +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._settings import Settings, ValidationSettings, create_settings +from bot.exts.filtering._utils import FieldRequiring, past_tense from bot.log import get_logger log = get_logger(__name__) class ListType(Enum): + """An enumeration of list types.""" + DENY = 0 ALLOW = 1 +class ListTypeConverter(Converter): + """A Converter to get the appropriate list type.""" + + aliases = ( + (ListType.DENY, {"deny", "blocklist", "blacklist", "denylist", "bl", "dl"}), + (ListType.ALLOW, {"allow", "allowlist", "whitelist", "al", "wl"}) + ) + + async def convert(self, ctx: Context, argument: str) -> ListType: + """Get the appropriate list type.""" + for list_type, aliases in self.aliases: + if argument in aliases or argument in map(past_tense, aliases): + return list_type + raise BadArgument(f"No matching list type found for {argument!r}.") + + class FilterList(FieldRequiring): """Dispatches events to lists of _filters, and aggregates the responses into a single list of actions to take.""" @@ -24,8 +44,8 @@ class FilterList(FieldRequiring): name = FieldRequiring.MUST_SET_UNIQUE def __init__(self, filter_type: Type[Filter]): - self._filter_lists: dict[ListType, list[Filter]] = {} - self._defaults: dict[ListType, dict[str, Settings]] = {} + self.filter_lists: dict[ListType, list[Filter]] = {} + self.defaults: dict[ListType, dict[str, Settings]] = {} self.filter_type = filter_type @@ -33,7 +53,7 @@ def add_list(self, list_data: Dict) -> None: """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" actions, validations = create_settings(list_data["settings"]) list_type = ListType(list_data["list_type"]) - self._defaults[list_type] = {"actions": actions, "validations": validations} + self.defaults[list_type] = {"actions": actions, "validations": validations} filters = [] for filter_data in list_data["filters"]: @@ -41,7 +61,7 @@ def add_list(self, list_data: Dict) -> None: filters.append(self.filter_type(filter_data, actions)) except TypeError as e: log.warning(e) - self._filter_lists[list_type] = filters + self.filter_lists[list_type] = filters @abstractmethod def triggers_for(self, ctx: FilterContext) -> list[Filter]: diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index 4495f44142..01e5861320 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -33,7 +33,7 @@ def triggers_for(self, ctx: FilterContext) -> list[Filter]: ctx = ctx.replace(content=text) return self.filter_list_result( - ctx, self._filter_lists[ListType.DENY], self._defaults[ListType.DENY]["validations"] + ctx, self.filter_lists[ListType.DENY], self.defaults[ListType.DENY]["validations"] ) @staticmethod diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 484e506fcf..e7fff20a6c 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -27,3 +27,10 @@ def __init__(self, filter_data: Dict, action_defaults: Optional[ActionSettings] @abstractmethod def triggered_on(self, ctx: FilterContext) -> bool: """Search for the filter's content within a given context.""" + + def __str__(self) -> str: + """A string representation of the filter.""" + string = f"#{self.id}. `{self.content}`" + if self.description: + string += f" - {self.description}" + return string diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py new file mode 100644 index 0000000000..95a840be87 --- /dev/null +++ b/bot/exts/filtering/_ui.py @@ -0,0 +1,48 @@ +from copy import copy + +import discord +import discord.ui +from discord.ext.commands import Context + +import bot +from bot.log import get_logger + +log = get_logger(__name__) + + +class ArgumentCompletionSelect(discord.ui.Select): + """A select detailing the options that can be picked to assign to a missing argument.""" + + def __init__(self, ctx: Context, arg_name: str, options: list[str]): + super().__init__( + placeholder=f"Select a value for {arg_name!r}", + options=[discord.SelectOption(label=option) for option in options] + ) + self.ctx = ctx + + async def callback(self, interaction: discord.Interaction) -> None: + """re-invoke the context command with the completed argument value.""" + await interaction.response.defer() + value = interaction.data['values'][0] + message = copy(self.ctx.message) + message.content = f'{message.content} "{value}"' + log.trace(f"Argument filled with the value {value}. Invoking {message.content!r}") + await bot.instance.process_commands(message) + + +class ArgumentCompletionView(discord.ui.View): + """A view used to complete a missing argument in an in invoked command.""" + + def __init__(self, ctx: Context, arg_name: str, options: list[str]): + super().__init__() + log.trace(f"The {arg_name} argument was designated missing in the invocation {ctx.view.buffer!r}") + self.add_item(ArgumentCompletionSelect(ctx, arg_name, options)) + self.ctx = ctx + + async def interaction_check(self, interaction: discord.Interaction) -> bool: + """Check to ensure that the interacting user is the user who invoked the command.""" + if interaction.user != self.ctx.author: + embed = discord.Embed(description="Sorry, but this dropdown menu can only be used by the original author.") + await interaction.response.send_message(embed=embed, ephemeral=True) + return False + return True diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index a769001f61..790f70ee5c 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -47,6 +47,17 @@ def clean_input(string: str) -> str: return INVISIBLE_RE.sub("", no_zalgo) +def past_tense(word: str) -> str: + """Return the past tense form of the input word.""" + if not word: + return word + if word.endswith("e"): + return word + "d" + if word.endswith("y") and len(word) > 1 and word[-2] not in "aeiou": + return word[:-1] + "ied" + return word + "ed" + + class FieldRequiring(ABC): """A mixin class that can force its concrete subclasses to set a value for specific class attributes.""" diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index c74b856988..58e16043a4 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -3,17 +3,21 @@ from functools import reduce from typing import Optional -from discord import Embed, HTTPException, Message -from discord.ext.commands import Cog +from discord import Colour, Embed, HTTPException, Message +from discord.ext import commands +from discord.ext.commands import BadArgument, Cog, Context, has_any_role from discord.utils import escape_markdown from bot.bot import Bot -from bot.constants import Colours, Webhooks +from bot.constants import Colours, MODERATION_ROLES, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext -from bot.exts.filtering._filter_lists import FilterList, filter_list_types +from bot.exts.filtering._filter_lists import FilterList, ListType, ListTypeConverter, filter_list_types from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings +from bot.exts.filtering._ui import ArgumentCompletionView +from bot.exts.filtering._utils import past_tense from bot.log import get_logger +from bot.pagination import LinePaginator from bot.utils.messages import format_channel, format_user log = get_logger(__name__) @@ -22,6 +26,8 @@ class Filtering(Cog): """Filtering and alerting for content posted on the server.""" + # region: init + def __init__(self, bot: Bot): self.bot = bot self.filter_lists: dict[str, FilterList] = {} @@ -54,7 +60,7 @@ async def cog_load(self) -> None: try: self.webhook = await self.bot.fetch_webhook(Webhooks.filters) except HTTPException: - log.error(f"Failed to fetch incidents webhook with id `{Webhooks.incidents}`.") + log.error(f"Failed to fetch incidents webhook with ID `{Webhooks.incidents}`.") def subscribe(self, filter_list: FilterList, *events: Event) -> None: """ @@ -73,21 +79,12 @@ def subscribe(self, filter_list: FilterList, *events: Event) -> None: if filter_list not in self._subscriptions[event]: self._subscriptions[event].append(filter_list) - async def _resolve_action( - self, ctx: FilterContext - ) -> tuple[dict[FilterList, list[Filter]], Optional[ActionSettings]]: - """Get the filters triggered per list, and resolve from them the action that needs to be taken for the event.""" - triggered = {} - for filter_list in self._subscriptions[ctx.event]: - triggered[filter_list] = filter_list.triggers_for(ctx) - - result_actions = None - if triggered: - result_actions = reduce( - operator.or_, (filter_.actions for filters in triggered.values() for filter_ in filters) - ) + async def cog_check(self, ctx: Context) -> bool: + """Only allow moderators to invoke the commands in this cog.""" + return await has_any_role(*MODERATION_ROLES).predicate(ctx) - return triggered, result_actions + # endregion + # region: listeners @Cog.listener() async def on_message(self, msg: Message) -> None: @@ -103,6 +100,100 @@ async def on_message(self, msg: Message) -> None: if ctx.send_alert: await self._send_alert(ctx, triggered) + # endregion + # region: blacklist commands + + @commands.group(aliases=("bl", "blacklist", "denylist", "dl")) + async def blocklist(self, ctx: Context) -> None: + """Group for managing blacklisted items.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @blocklist.command(name="list", aliases=("get",)) + async def bl_list(self, ctx: Context, list_name: Optional[str] = None) -> None: + """List the contents of a specified blacklist.""" + if list_name is None: + await ctx.send( + "The **list_name** argument is unspecified. Please pick a value from the options below:", + view=ArgumentCompletionView(ctx, "list_name", list(self.filter_lists)) + ) + return + await self._send_list(ctx, list_name, ListType.DENY) + + # endregion + # region: whitelist commands + + @commands.group(aliases=("wl", "whitelist", "al")) + async def allowlist(self, ctx: Context) -> None: + """Group for managing blacklisted items.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @allowlist.command(name="list", aliases=("get",)) + async def al_list(self, ctx: Context, list_name: Optional[str] = None) -> None: + """List the contents of a specified whitelist.""" + if list_name is None: + await ctx.send( + "The **list_name** argument is unspecified. Please pick a value from the options below:", + view=ArgumentCompletionView(ctx, "list_name", list(self.filter_lists)) + ) + return + await self._send_list(ctx, list_name, ListType.ALLOW) + + # endregion + # region: filter commands + + @commands.group(aliases=("filters", "f")) + async def filter(self, ctx: Context) -> None: + """Group for managing filters.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @filter.command(name="list", aliases=("get",)) + async def f_list( + self, ctx: Context, list_type: Optional[ListTypeConverter] = None, list_name: Optional[str] = None + ) -> None: + """List the contents of a specified list of filters.""" + if list_name is None: + await ctx.send( + "The **list_name** argument is unspecified. Please pick a value from the options below:", + view=ArgumentCompletionView(ctx, "list_name", list(self.filter_lists)) + ) + return + + if list_type is None: + filter_list = self._get_list_by_name(list_name) + if len(filter_list.filter_lists) > 1: + await ctx.send( + "The **list_type** argument is unspecified. Please pick a value from the options below:", + view=ArgumentCompletionView(ctx, "list_type", [option.name for option in ListType]) + ) + return + list_type = list(filter_list.filter_lists)[0] + + await self._send_list(ctx, list_name, list_type) + + # endregion + # region: helper functions + + async def _resolve_action( + self, ctx: FilterContext + ) -> tuple[dict[FilterList, list[Filter]], Optional[ActionSettings]]: + """Get the filters triggered per list, and resolve from them the action that needs to be taken for the event.""" + triggered = {} + for filter_list in self._subscriptions[ctx.event]: + result = filter_list.triggers_for(ctx) + if result: + triggered[filter_list] = result + + result_actions = None + if triggered: + result_actions = reduce( + operator.or_, (filter_.actions for filters in triggered.values() for filter_ in filters) + ) + + return triggered, result_actions + async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, list[Filter]]) -> None: """Build an alert message from the filter context, and send it via the alert webhook.""" if not self.webhook: @@ -144,6 +235,32 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds]) + def _get_list_by_name(self, list_name: str) -> FilterList: + """Get a filter list by its name, or raise an error if there's no such list.""" + log.trace(f"Getting the filter list matching the name {list_name}") + filter_list = self.filter_lists.get(list_name) + if not filter_list: + if list_name.endswith("s"): # The user may have attempted to use the plural form. + filter_list = self.filter_lists.get(list_name[:-1]) + if not filter_list: + raise BadArgument(f"There's no filter list named {list_name!r}.") + log.trace(f"Found list named {filter_list.name}") + return filter_list + + async def _send_list(self, ctx: Context, list_name: str, list_type: ListType) -> None: + """Show the list of filters identified by the list name and type.""" + filter_list = self._get_list_by_name(list_name) + lines = list(map(str, filter_list.filter_lists.get(list_type, []))) + log.trace(f"Sending a list of {len(lines)} filters.") + + list_name_plural = list_name + ("s" if not list_name.endswith("s") else "") + embed = Embed(colour=Colour.blue()) + embed.set_author(name=f"List of {past_tense(list_type.name.lower())} {list_name_plural} ({len(lines)} total)") + + await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) + + # endregion + async def setup(bot: Bot) -> None: """Load the Filtering cog.""" From d1ae7ce9235e4d63ee1dde282ca890ac5509f950 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 22 Feb 2022 22:15:19 +0200 Subject: [PATCH 004/132] Accept strings in channel scope and change role string interpretation The channel scope settings were changed to accomodate strings. That means that if a string is specified, the bot will look whether the context channel's name matches. If it's a number, it will match the ID. Accordingly the same changed was applied to the bypass roles and pings settings: if it's a non-numeric string, it will look for a role with that name. --- .../filtering/_settings_types/bypass_roles.py | 13 ++++----- .../_settings_types/channel_scope.py | 27 ++++++++++++++----- bot/exts/filtering/_settings_types/ping.py | 26 ++++++++++-------- bot/exts/filtering/_utils.py | 8 ------ bot/exts/filtering/filtering.py | 2 +- .../exts/filtering/test_settings_entries.py | 8 +++--- tests/helpers.py | 2 +- 7 files changed, 48 insertions(+), 38 deletions(-) diff --git a/bot/exts/filtering/_settings_types/bypass_roles.py b/bot/exts/filtering/_settings_types/bypass_roles.py index 9665283ffd..bfc4a30fd0 100644 --- a/bot/exts/filtering/_settings_types/bypass_roles.py +++ b/bot/exts/filtering/_settings_types/bypass_roles.py @@ -4,7 +4,6 @@ from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ValidationEntry -from bot.exts.filtering._utils import ROLE_LITERALS class RoleBypass(ValidationEntry): @@ -16,14 +15,16 @@ def __init__(self, entry_data: Any): super().__init__(entry_data) self.roles = set() for role in entry_data: - if role in ROLE_LITERALS: - self.roles.add(ROLE_LITERALS[role]) - elif role.isdigit(): + if role.isdigit(): self.roles.add(int(role)) - # Ignore entries that can't be resolved. + else: + self.roles.add(role) def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered on this user given their roles.""" if not isinstance(ctx.author, Member): return True - return all(member_role.id not in self.roles for member_role in ctx.author.roles) + return all( + member_role.id not in self.roles and member_role.name not in self.roles + for member_role in ctx.author.roles + ) diff --git a/bot/exts/filtering/_settings_types/channel_scope.py b/bot/exts/filtering/_settings_types/channel_scope.py index b17914f2ff..63da6c7e50 100644 --- a/bot/exts/filtering/_settings_types/channel_scope.py +++ b/bot/exts/filtering/_settings_types/channel_scope.py @@ -1,9 +1,16 @@ -from typing import Any +from typing import Any, Union from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ValidationEntry +def maybe_cast_to_int(item: str) -> Union[str, int]: + """Cast the item to int if it consists of only digit, or leave as is otherwise.""" + if item.isdigit(): + return int(item) + return item + + class ChannelScope(ValidationEntry): """A setting entry which tells whether the filter was invoked in a whitelisted channel or category.""" @@ -12,17 +19,17 @@ class ChannelScope(ValidationEntry): def __init__(self, entry_data: Any): super().__init__(entry_data) if entry_data["disabled_channels"]: - self.disabled_channels = set(entry_data["disabled_channels"]) + self.disabled_channels = set(map(maybe_cast_to_int, entry_data["disabled_channels"])) else: self.disabled_channels = set() if entry_data["disabled_categories"]: - self.disabled_categories = set(entry_data["disabled_categories"]) + self.disabled_categories = set(map(maybe_cast_to_int, entry_data["disabled_categories"])) else: self.disabled_categories = set() if entry_data["enabled_channels"]: - self.enabled_channels = set(entry_data["enabled_channels"]) + self.enabled_channels = set(map(maybe_cast_to_int, entry_data["enabled_channels"])) else: self.enabled_channels = set() @@ -34,12 +41,18 @@ def triggers_on(self, ctx: FilterContext) -> bool: If the channel is explicitly enabled, it bypasses the set disabled channels and categories. """ channel = ctx.channel - if hasattr(channel, "parent"): - channel = channel.parent - return ( + enabled_id = ( channel.id in self.enabled_channels or ( channel.id not in self.disabled_channels and (not channel.category or channel.category.id not in self.disabled_categories) ) ) + enabled_name = ( + channel.name in self.enabled_channels + or ( + channel.name not in self.disabled_channels + and (not channel.category or channel.category.name not in self.disabled_categories) + ) + ) + return enabled_id and enabled_name diff --git a/bot/exts/filtering/_settings_types/ping.py b/bot/exts/filtering/_settings_types/ping.py index 857e4a7e8f..0f9a014c4e 100644 --- a/bot/exts/filtering/_settings_types/ping.py +++ b/bot/exts/filtering/_settings_types/ping.py @@ -5,7 +5,6 @@ from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry -from bot.exts.filtering._utils import ROLE_LITERALS class Ping(ActionEntry): @@ -40,13 +39,18 @@ def _resolve_mention(mention: str, guild: Guild) -> str: """Return the appropriate formatting for the formatting, be it a literal, a user ID, or a role ID.""" if mention in ("here", "everyone"): return f"@{mention}" - if mention in ROLE_LITERALS: - return f"<@&{ROLE_LITERALS[mention]}>" - if not mention.isdigit(): - return mention - - mention = int(mention) - if any(mention == role.id for role in guild.roles): - return f"<@&{mention}>" - else: - return f"<@{mention}>" + if mention.isdigit(): # It's an ID. + mention = int(mention) + if any(mention == role.id for role in guild.roles): + return f"<@&{mention}>" + else: + return f"<@{mention}>" + + # It's a name + for role in guild.roles: + if role.name == mention: + return role.mention + for member in guild.members: + if str(member) == mention: + return member.mention + return mention diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index 790f70ee5c..d092621938 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -8,14 +8,6 @@ import regex -from bot.constants import Roles - -ROLE_LITERALS = { - "admins": Roles.admins, - "onduty": Roles.moderators, - "staff": Roles.helpers -} - VARIATION_SELECTORS = r"\uFE00-\uFE0F\U000E0100-\U000E01EF" INVISIBLE_RE = regex.compile(rf"[{VARIATION_SELECTORS}\p{{UNASSIGNED}}\p{{FORMAT}}\p{{CONTROL}}--\s]", regex.V1) ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIATION_SELECTORS}]]", regex.V1) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 58e16043a4..d34b4928a0 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -89,7 +89,7 @@ async def cog_check(self, ctx: Context) -> bool: @Cog.listener() async def on_message(self, msg: Message) -> None: """Filter the contents of a sent message.""" - if msg.author.bot: + if msg.author.bot or msg.webhook_id: return ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py index 4db6438abc..d18861bd6f 100644 --- a/tests/bot/exts/filtering/test_settings_entries.py +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -62,7 +62,7 @@ def test_context_doesnt_trigger_for_empty_channel_scope(self): def test_context_doesnt_trigger_for_disabled_channel(self): """A filter shouldn't trigger if it's been disabled in the channel.""" channel = MockTextChannel(id=123) - scope = ChannelScope({"disabled_channels": [123], "disabled_categories": None, "enabled_channels": None}) + scope = ChannelScope({"disabled_channels": ["123"], "disabled_categories": None, "enabled_channels": None}) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -71,9 +71,9 @@ def test_context_doesnt_trigger_for_disabled_channel(self): def test_context_doesnt_trigger_in_disabled_category(self): """A filter shouldn't trigger if it's been disabled in the category.""" - channel = MockTextChannel() + channel = MockTextChannel(category=MockCategoryChannel(id=456)) scope = ChannelScope({ - "disabled_channels": None, "disabled_categories": [channel.category.id], "enabled_channels": None + "disabled_channels": None, "disabled_categories": ["456"], "enabled_channels": None }) self.ctx.channel = channel @@ -84,7 +84,7 @@ def test_context_doesnt_trigger_in_disabled_category(self): def test_context_triggers_in_enabled_channel_in_disabled_category(self): """A filter should trigger in an enabled channel even if it's been disabled in the category.""" channel = MockTextChannel(id=123, category=MockCategoryChannel(id=234)) - scope = ChannelScope({"disabled_channels": None, "disabled_categories": [234], "enabled_channels": [123]}) + scope = ChannelScope({"disabled_channels": None, "disabled_categories": ["234"], "enabled_channels": ["123"]}) self.ctx.channel = channel result = scope.triggers_on(self.ctx) diff --git a/tests/helpers.py b/tests/helpers.py index 17214553ce..e74306d23a 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -423,7 +423,7 @@ def __init__(self, **kwargs) -> None: class MockCategoryChannel(CustomMockMixin, unittest.mock.Mock, HashableMixin): def __init__(self, **kwargs) -> None: default_kwargs = {'id': next(self.discord_id)} - super().__init__(**collections.ChainMap(default_kwargs, kwargs)) + super().__init__(**collections.ChainMap(kwargs, default_kwargs)) # Create a Message instance to get a realistic MagicMock of `discord.Message` From 555be653507baeda16f069114dbf5e7a2752d6e3 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 24 Feb 2022 21:20:37 +0200 Subject: [PATCH 005/132] Add file extension filtering This commmit migrates the AntiMalware cog to a new filter list which goes over a message's attachments. Some changes were needed to accomodate the new list, primarily what a filter list returns for a given context: Instead of returning a list of filters, it will return the action itself that should be taken. This adds the flexibility of not needing existing filters to dictate the action. For example, in the case of the extensions list, an action should be taken when filters were *not* triggered. Or more precisely, when not all attachment extensions are whitelisted. Therefore, the action in that case is dictated by the filter list (stored as the list's default actions). Additionally each filter list can now return its own message for the alert embed, instead of the cog formatting it according to the filters raised. Because again, an action might be taken without any deny filters being triggered. This is going to be especially relevant for the invites list. Additionally, the infraction_and_notification action now doesn't redirect the notification to the context channel when the DM fails, since this can be incredibly noisy in cases of spam. If we want this functionality, a more suitable solution should be found. --- bot/exts/filtering/_filter_context.py | 6 +- bot/exts/filtering/_filter_lists/extension.py | 92 +++++++++++++++++++ .../filtering/_filter_lists/filter_list.py | 10 +- bot/exts/filtering/_filter_lists/token.py | 24 ++++- bot/exts/filtering/_filters/extension.py | 10 ++ .../infraction_and_notification.py | 14 ++- bot/exts/filtering/filtering.py | 56 ++++++----- 7 files changed, 163 insertions(+), 49 deletions(-) create mode 100644 bot/exts/filtering/_filter_lists/extension.py create mode 100644 bot/exts/filtering/_filters/extension.py diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index ee9e87f56e..ad5c8636ff 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -4,7 +4,7 @@ from enum import Enum, auto from typing import Optional, Union -from discord import DMChannel, Embed, Message, TextChannel, Thread, User +from discord import DMChannel, Message, TextChannel, Thread, User class Event(Enum): @@ -22,12 +22,12 @@ class FilterContext: event: Event # The type of event author: User # Who triggered the event channel: Union[TextChannel, Thread, DMChannel] # The channel involved - content: str # What actually needs filtering + content: Union[str, set[str]] # What actually needs filtering message: Optional[Message] # The message involved embeds: list = field(default_factory=list) # Any embeds involved # Output context dm_content: str = field(default_factory=str) # The content to DM the invoker - dm_embed: Embed = field(default_factory=Embed) # The embed to DM the invoker + dm_embed: str = field(default_factory=str) # The embed description to DM the invoker send_alert: bool = field(default=True) # Whether to send an alert for the moderators alert_content: str = field(default_factory=str) # The content of the alert alert_embeds: list = field(default_factory=list) # Any embeds to add to the alert diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py new file mode 100644 index 0000000000..c55cda114d --- /dev/null +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import typing +from os.path import splitext +from typing import Optional + +import bot +from bot.constants import Channels, URLs +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType +from bot.exts.filtering._filters.extension import ExtensionFilter +from bot.exts.filtering._settings import ActionSettings + +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + + +PY_EMBED_DESCRIPTION = ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" +) + +TXT_LIKE_FILES = {".txt", ".csv", ".json"} +TXT_EMBED_DESCRIPTION = ( + "You either uploaded a `{blocked_extension}` file or entered a message that was too long. " + f"Please use our [paste bin]({URLs.site_schema}{URLs.site_paste}) instead." +) + +DISALLOWED_EMBED_DESCRIPTION = ( + "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " + "We currently allow the following file types: **{joined_whitelist}**.\n\n" + "Feel free to ask in {meta_channel_mention} if you think this is a mistake." +) + + +class ExtensionsList(FilterList): + """A list of filters, each looking for an attachment with a specific extension.""" + + name = "extension" + + def __init__(self, filtering_cog: Filtering): + super().__init__(ExtensionFilter) + filtering_cog.subscribe(self, Event.MESSAGE) + self._whitelisted_description = None + + def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + # Return early if the message doesn't have attachments. + if not ctx.message.attachments: + return None, "" + + # Find all extensions in the message. + all_ext = { + (splitext(attachment.filename.lower())[1], attachment.filename) for attachment in ctx.message.attachments + } + new_ctx = ctx.replace(content={ext for ext, _ in all_ext}) # And prepare the context for the filters to read. + triggered = self.filter_list_result( + new_ctx, self.filter_lists[ListType.ALLOW], self.defaults[ListType.ALLOW]["validations"] + ) + allowed_ext = {filter_.content for filter_ in triggered} # Get the extensions in the message that are allowed. + + # See if there are any extensions left which aren't allowed. + not_allowed = {ext: filename for ext, filename in all_ext if ext not in allowed_ext} + + if not not_allowed: # Yes, it's a double negative. Meaning all attachments are allowed :) + return None, "" + + # Something is disallowed. + if ".py" in not_allowed: + # Provide a pastebin link for .py files. + ctx.dm_embed = PY_EMBED_DESCRIPTION + elif txt_extensions := {ext for ext in TXT_LIKE_FILES if ext in not_allowed}: + # Work around Discord auto-conversion of messages longer than 2000 chars to .txt + cmd_channel = bot.instance.get_channel(Channels.bot_commands) + ctx.dm_embed = TXT_EMBED_DESCRIPTION.format( + blocked_extension=txt_extensions.pop(), + cmd_channel_mention=cmd_channel.mention + ) + else: + meta_channel = bot.instance.get_channel(Channels.meta) + if not self._whitelisted_description: + self._whitelisted_description = ', '.join( + filter_.content for filter_ in self.filter_lists[ListType.ALLOW] + ) + ctx.dm_embed = DISALLOWED_EMBED_DESCRIPTION.format( + joined_whitelist=self._whitelisted_description, + blocked_extensions_str=", ".join(not_allowed), + meta_channel_mention=meta_channel.mention, + ) + + ctx.matches += not_allowed.values() + return self.defaults[ListType.ALLOW]["actions"], ", ".join(f"`{ext}`" for ext in not_allowed) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 1060f11dba..9fb1443541 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,12 +1,12 @@ from abc import abstractmethod from enum import Enum -from typing import Dict, List, Type +from typing import Dict, List, Optional, Type from discord.ext.commands import BadArgument, Context, Converter from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter -from bot.exts.filtering._settings import Settings, ValidationSettings, create_settings +from bot.exts.filtering._settings import ActionSettings, ValidationSettings, create_settings from bot.exts.filtering._utils import FieldRequiring, past_tense from bot.log import get_logger @@ -45,7 +45,7 @@ class FilterList(FieldRequiring): def __init__(self, filter_type: Type[Filter]): self.filter_lists: dict[ListType, list[Filter]] = {} - self.defaults: dict[ListType, dict[str, Settings]] = {} + self.defaults = {} self.filter_type = filter_type @@ -64,8 +64,8 @@ def add_list(self, list_data: Dict) -> None: self.filter_lists[list_type] = filters @abstractmethod - def triggers_for(self, ctx: FilterContext) -> list[Filter]: - """Dispatch the given event to the list's filters, and return filters triggered.""" + def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" @staticmethod def filter_list_result(ctx: FilterContext, filters: List[Filter], defaults: ValidationSettings) -> list[Filter]: diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index 01e5861320..d4eb10591f 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -2,11 +2,14 @@ import re import typing +from functools import reduce +from operator import or_ +from typing import Optional from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType -from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._filters.token import TokenFilter +from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._utils import clean_input if typing.TYPE_CHECKING: @@ -24,17 +27,30 @@ def __init__(self, filtering_cog: Filtering): super().__init__(TokenFilter) filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) - def triggers_for(self, ctx: FilterContext) -> list[Filter]: - """Dispatch the given event to the list's filters, and return filters triggered.""" + def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" text = ctx.content + if not text: + return None, "" if SPOILER_RE.search(text): text = self._expand_spoilers(text) text = clean_input(text) ctx = ctx.replace(content=text) - return self.filter_list_result( + triggers = self.filter_list_result( ctx, self.filter_lists[ListType.DENY], self.defaults[ListType.DENY]["validations"] ) + actions = None + message = "" + if triggers: + actions = reduce(or_, (filter_.actions for filter_ in triggers)) + if len(triggers) == 1: + message = f"#{triggers[0].id} (`{triggers[0].content}`)" + if triggers[0].description: + message += f" - {triggers[0].description}" + else: + message = ", ".join(f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers) + return actions, message @staticmethod def _expand_spoilers(text: str) -> str: diff --git a/bot/exts/filtering/_filters/extension.py b/bot/exts/filtering/_filters/extension.py new file mode 100644 index 0000000000..85bfd05b2b --- /dev/null +++ b/bot/exts/filtering/_filters/extension.py @@ -0,0 +1,10 @@ +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filters.filter import Filter + + +class ExtensionFilter(Filter): + """A filter which looks for a specific attachment extension in messages.""" + + def triggered_on(self, ctx: FilterContext) -> bool: + """Searches for an attachment extension in the context content, given as a set of extensions.""" + return self.content in ctx.content diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index 263fd851ca..68ffa166f6 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -4,7 +4,7 @@ from typing import Any, Optional import arrow -from discord import Colour +from discord import Colour, Embed from discord.errors import Forbidden import bot @@ -74,22 +74,20 @@ async def action(self, ctx: FilterContext) -> None: # If there is no infraction to apply, any DM contents already provided in the context take precedence. if self.infraction_type == Infraction.NONE and (ctx.dm_content or ctx.dm_embed): dm_content = ctx.dm_content - dm_embed = ctx.dm_embed.description + dm_embed = ctx.dm_embed else: dm_content = self.dm_content dm_embed = self.dm_embed if dm_content or dm_embed: dm_content = f"Hey {ctx.author.mention}!\n{dm_content}" - ctx.dm_embed.description = dm_embed - if not ctx.dm_embed.colour: - ctx.dm_embed.colour = Colour.og_blurple() + dm_embed = Embed(description=dm_embed, colour=Colour.og_blurple()) try: - await ctx.author.send(dm_content, embed=ctx.dm_embed) + await ctx.author.send(dm_content, embed=dm_embed) + ctx.action_descriptions.append("notified") except Forbidden: - await ctx.channel.send(ctx.dm_content, embed=ctx.dm_embed) - ctx.action_descriptions.append("notified") + ctx.action_descriptions.append("notified (failed)") msg_ctx = await bot.instance.get_context(ctx.message) msg_ctx.guild = bot.instance.get_guild(Guild.id) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index d34b4928a0..c22e7316fe 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -12,7 +12,6 @@ from bot.constants import Colours, MODERATION_ROLES, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, ListTypeConverter, filter_list_types -from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._ui import ArgumentCompletionView from bot.exts.filtering._utils import past_tense @@ -94,11 +93,11 @@ async def on_message(self, msg: Message) -> None: ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) - triggered, result_actions = await self._resolve_action(ctx) + result_actions, list_messages = await self._resolve_action(ctx) if result_actions: await result_actions.action(ctx) - if ctx.send_alert: - await self._send_alert(ctx, triggered) + if ctx.send_alert: + await self._send_alert(ctx, list_messages) # endregion # region: blacklist commands @@ -178,23 +177,29 @@ async def f_list( async def _resolve_action( self, ctx: FilterContext - ) -> tuple[dict[FilterList, list[Filter]], Optional[ActionSettings]]: - """Get the filters triggered per list, and resolve from them the action that needs to be taken for the event.""" - triggered = {} + ) -> tuple[Optional[ActionSettings], dict[FilterList, str]]: + """ + Return the actions that should be taken for all filter lists in the given context. + + Additionally, a message is possibly provided from each filter list describing the triggers, + which should be relayed to the moderators. + """ + actions = [] + messages = {} for filter_list in self._subscriptions[ctx.event]: - result = filter_list.triggers_for(ctx) - if result: - triggered[filter_list] = result + list_actions, list_message = filter_list.actions_for(ctx) + if list_actions: + actions.append(list_actions) + if list_message: + messages[filter_list] = list_message result_actions = None - if triggered: - result_actions = reduce( - operator.or_, (filter_.actions for filters in triggered.values() for filter_ in filters) - ) + if actions: + result_actions = reduce(operator.or_, (action for action in actions)) - return triggered, result_actions + return result_actions, messages - async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, list[Filter]]) -> None: + async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, str]) -> None: """Build an alert message from the filter context, and send it via the alert webhook.""" if not self.webhook: return @@ -208,19 +213,12 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}" else: triggered_in = "**DM**" - if len(triggered_filters) == 1 and len(list(triggered_filters.values())[0]) == 1: - filter_list, (filter_,) = next(iter(triggered_filters.items())) - filters = f"**{filter_list.name.title()} Filter:** #{filter_.id} (`{filter_.content}`)" - if filter_.description: - filters += f" - {filter_.description}" - else: - filters = [] - for filter_list, list_filters in triggered_filters.items(): - filters.append( - (f"**{filter_list.name.title()} Filters:** " - ", ".join(f"#{filter_.id} (`{filter_.content}`)" for filter_ in list_filters)) - ) - filters = "\n".join(filters) + + filters = [] + for filter_list, list_message in triggered_filters.items(): + if list_message: + filters.append(f"**{filter_list.name.title()} Filters:** {list_message}") + filters = "\n".join(filters) matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) actions = "**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") From d763305d02a1bfd3cae1af3533837dc9dfb7966b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 26 Feb 2022 22:03:56 +0200 Subject: [PATCH 006/132] Add guild invite filtering This commit adds the invite filtering implementation to the new system. This also fixes an issue with the implementation of the extension filtering, where there was no way to tell the bot to ignore a user when they posted a non-whitelisted file extension, since there's no relevant filter in this scenario. Instead the extensions and invites filters now use the whitelist validation defaults to dictate when filtering should be done at all. For example, if the list validations are to ignore Helpers, then no invite filtering will occur no matter what. The meaning of this is that the system is somewhat less configurable, because settings overrides in filters belonging to a whitelist are meaningless. Additionally this commit contains the following fixes: - If the user tries to show the filters in a list which doesn't exist, it says it doesn't exist instead of saying there are 0 filters. - The filter context content is `Union[str, set]` instead of `Union[str, set[str]]`. - An empty embed will no longer be created when `dm_embed` is empty. --- bot/exts/filtering/_filter_context.py | 4 +- bot/exts/filtering/_filter_lists/extension.py | 10 +- .../filtering/_filter_lists/filter_list.py | 2 +- bot/exts/filtering/_filter_lists/invite.py | 114 ++++++++++++++++++ bot/exts/filtering/_filter_lists/token.py | 2 +- bot/exts/filtering/_filters/filter.py | 4 +- bot/exts/filtering/_filters/invite.py | 17 +++ .../infraction_and_notification.py | 2 +- bot/exts/filtering/filtering.py | 18 +-- 9 files changed, 154 insertions(+), 19 deletions(-) create mode 100644 bot/exts/filtering/_filter_lists/invite.py create mode 100644 bot/exts/filtering/_filters/invite.py diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index ad5c8636ff..2fec9ce428 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -22,13 +22,13 @@ class FilterContext: event: Event # The type of event author: User # Who triggered the event channel: Union[TextChannel, Thread, DMChannel] # The channel involved - content: Union[str, set[str]] # What actually needs filtering + content: Union[str, set] # What actually needs filtering message: Optional[Message] # The message involved embeds: list = field(default_factory=list) # Any embeds involved # Output context dm_content: str = field(default_factory=str) # The content to DM the invoker dm_embed: str = field(default_factory=str) # The embed description to DM the invoker - send_alert: bool = field(default=True) # Whether to send an alert for the moderators + send_alert: bool = field(default=False) # Whether to send an alert for the moderators alert_content: str = field(default_factory=str) # The content of the alert alert_embeds: list = field(default_factory=list) # Any embeds to add to the alert action_descriptions: list = field(default_factory=list) # What actions were taken diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index c55cda114d..ceb8bb9580 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -43,20 +43,22 @@ def __init__(self, filtering_cog: Filtering): filtering_cog.subscribe(self, Event.MESSAGE) self._whitelisted_description = None - def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" # Return early if the message doesn't have attachments. if not ctx.message.attachments: return None, "" + _, failed = self.defaults[ListType.ALLOW]["validations"].evaluate(ctx) + if failed: # There's no extension filtering in this context. + return None, "" + # Find all extensions in the message. all_ext = { (splitext(attachment.filename.lower())[1], attachment.filename) for attachment in ctx.message.attachments } new_ctx = ctx.replace(content={ext for ext, _ in all_ext}) # And prepare the context for the filters to read. - triggered = self.filter_list_result( - new_ctx, self.filter_lists[ListType.ALLOW], self.defaults[ListType.ALLOW]["validations"] - ) + triggered = [filter_ for filter_ in self.filter_lists[ListType.ALLOW] if filter_.triggered_on(new_ctx)] allowed_ext = {filter_.content for filter_ in triggered} # Get the extensions in the message that are allowed. # See if there are any extensions left which aren't allowed. diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 9fb1443541..672811444c 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -64,7 +64,7 @@ def add_list(self, list_data: Dict) -> None: self.filter_lists[list_type] = filters @abstractmethod - def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" @staticmethod diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py new file mode 100644 index 0000000000..04afff0f71 --- /dev/null +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import typing +from functools import reduce +from operator import or_ +from typing import Optional + +from botcore.regex import DISCORD_INVITE +from discord import Embed, Invite +from discord.errors import NotFound + +import bot +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType +from bot.exts.filtering._filters.invite import InviteFilter +from bot.exts.filtering._settings import ActionSettings +from bot.exts.filtering._utils import clean_input + +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + + +class InviteList(FilterList): + """A list of filters, each looking for guild invites to a specific guild.""" + + name = "invite" + + def __init__(self, filtering_cog: Filtering): + super().__init__(InviteFilter) + filtering_cog.subscribe(self, Event.MESSAGE) + + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + _, failed = self.defaults[ListType.ALLOW]["validations"].evaluate(ctx) + if failed: # There's no invite filtering in this context. + return None, "" + + text = clean_input(ctx.content) + + # Avoid escape characters + text = text.replace("\\", "") + + matches = list(DISCORD_INVITE.finditer(text)) + invite_codes = {m.group("invite") for m in matches} + if not invite_codes: + return None, "" + + # Sort the invites into three categories: + denied_by_default = dict() # Denied unless whitelisted. + allowed_by_default = dict() # Allowed unless blacklisted (partnered or verified servers). + disallowed_invites = dict() # Always denied (invalid invites). + for invite_code in invite_codes: + try: + invite = await bot.instance.fetch_invite(invite_code) + except NotFound: + disallowed_invites[invite_code] = None + else: + if not invite.guild: + disallowed_invites[invite_code] = invite + else: + if "PARTNERED" in invite.guild.features or "VERIFIED" in invite.guild.features: + allowed_by_default[invite_code] = invite + else: + denied_by_default[invite_code] = invite + + # Add the disallowed by default unless they're whitelisted. + guilds_for_inspection = {invite.guild.id for invite in denied_by_default.values()} + new_ctx = ctx.replace(content=guilds_for_inspection) + allowed = {filter_.content for filter_ in self.filter_lists[ListType.ALLOW] if filter_.triggered_on(new_ctx)} + disallowed_invites.update({ + invite_code: invite for invite_code, invite in denied_by_default.items() if invite.guild.id not in allowed + }) + + # Add the allowed by default only if they're blacklisted. + guilds_for_inspection = {invite.guild.id for invite in allowed_by_default.values()} + new_ctx = ctx.replace(content=guilds_for_inspection) + triggered = self.filter_list_result( + new_ctx, self.filter_lists[ListType.ALLOW], self.defaults[ListType.DENY]["validations"] + ) + disallowed_invites.update({ + invite_code: invite for invite_code, invite in allowed_by_default.items() + if invite.guild.id in {filter_.content for filter_ in triggered} + }) + + if not disallowed_invites: + return None, "" + + actions = None + if len(disallowed_invites) > len(triggered): # There are invites which weren't allowed but aren't blacklisted. + actions = reduce(or_, (filter_.actions for filter_ in triggered), self.defaults[ListType.ALLOW]["actions"]) + elif triggered: + actions = reduce(or_, (filter_.actions for filter_ in triggered)) + ctx.matches += {match[0] for match in matches if match.group("invite") in disallowed_invites} + ctx.alert_embeds += (self._guild_embed(invite) for invite in disallowed_invites.values() if invite) + return actions, ", ".join(f"`{invite}`" for invite in disallowed_invites) + + @staticmethod + def _guild_embed(invite: Invite) -> Embed: + """Return an embed representing the guild invites to.""" + embed = Embed() + if invite.guild: + embed.title = invite.guild.name + embed.set_thumbnail(url=invite.guild.icon.url) + embed.set_footer(text=f"Guild ID: {invite.guild.id}") + else: + embed.title = "Group DM" + + embed.description = ( + f"**Invite Code:** {invite.code}\n" + f"**Members:** {invite.approximate_member_count}\n" + f"**Active:** {invite.approximate_presence_count}" + ) + + return embed diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index d4eb10591f..c232b55e5b 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -27,7 +27,7 @@ def __init__(self, filtering_cog: Filtering): super().__init__(TokenFilter) filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) - def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" text = ctx.content if not text: diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index e7fff20a6c..f1e5eac916 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Dict, Optional +from typing import Optional from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings import ActionSettings, create_settings @@ -13,7 +13,7 @@ class Filter(ABC): and defines what action should be performed if it is triggered. """ - def __init__(self, filter_data: Dict, action_defaults: Optional[ActionSettings] = None): + def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): self.id = filter_data["id"] self.content = filter_data["content"] self.description = filter_data["description"] diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py new file mode 100644 index 0000000000..afe4fdd945 --- /dev/null +++ b/bot/exts/filtering/_filters/invite.py @@ -0,0 +1,17 @@ +from typing import Optional + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._settings import ActionSettings + + +class InviteFilter(Filter): + """A filter which looks for invites to a specific guild in messages.""" + + def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): + super().__init__(filter_data, action_defaults) + self.content = int(self.content) + + def triggered_on(self, ctx: FilterContext) -> bool: + """Searches for a guild ID in the context content, given as a set of IDs.""" + return self.content in ctx.content diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index 68ffa166f6..d308bf444c 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -81,7 +81,7 @@ async def action(self, ctx: FilterContext) -> None: if dm_content or dm_embed: dm_content = f"Hey {ctx.author.mention}!\n{dm_content}" - dm_embed = Embed(description=dm_embed, colour=Colour.og_blurple()) + dm_embed = Embed(description=dm_embed, colour=Colour.og_blurple()) if dm_embed else None try: await ctx.author.send(dm_content, embed=dm_embed) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index c22e7316fe..5eefdf4e47 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -175,9 +175,7 @@ async def f_list( # endregion # region: helper functions - async def _resolve_action( - self, ctx: FilterContext - ) -> tuple[Optional[ActionSettings], dict[FilterList, str]]: + async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], dict[FilterList, str]]: """ Return the actions that should be taken for all filter lists in the given context. @@ -187,7 +185,7 @@ async def _resolve_action( actions = [] messages = {} for filter_list in self._subscriptions[ctx.event]: - list_actions, list_message = filter_list.actions_for(ctx) + list_actions, list_message = await filter_list.actions_for(ctx) if list_actions: actions.append(list_actions) if list_message: @@ -231,7 +229,7 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi embed_content = embed_content[:4000] + " [...]" embed.description = embed_content - await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds]) + await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10]) def _get_list_by_name(self, list_name: str) -> FilterList: """Get a filter list by its name, or raise an error if there's no such list.""" @@ -248,12 +246,16 @@ def _get_list_by_name(self, list_name: str) -> FilterList: async def _send_list(self, ctx: Context, list_name: str, list_type: ListType) -> None: """Show the list of filters identified by the list name and type.""" filter_list = self._get_list_by_name(list_name) - lines = list(map(str, filter_list.filter_lists.get(list_type, []))) + type_filters = filter_list.filter_lists.get(list_type) + if type_filters is None: + await ctx.send(f":x: There is no list of {past_tense(list_type.name.lower())} {filter_list.name}s.") + return + + lines = list(map(str, type_filters)) log.trace(f"Sending a list of {len(lines)} filters.") - list_name_plural = list_name + ("s" if not list_name.endswith("s") else "") embed = Embed(colour=Colour.blue()) - embed.set_author(name=f"List of {past_tense(list_type.name.lower())} {list_name_plural} ({len(lines)} total)") + embed.set_author(name=f"List of {past_tense(list_type.name.lower())} {filter_list.name}s ({len(lines)} total)") await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) From 08d06555d28a9588d9d8c814655df9c55c0c5563 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 27 Feb 2022 00:06:56 +0200 Subject: [PATCH 007/132] Fix argument completion for non-last args Previously the completed arg would be appended to the end of the message, which didn't work if the missing argument wasn't the last one (for example, a value was given to a following argument because of a failed conversion for previous ones). The select now employs a different strategy of storing the provided args, and the position where the new are should be inserted. --- bot/exts/filtering/_filter_lists/__init__.py | 4 +- .../filtering/_filter_lists/filter_list.py | 26 ++++++------ bot/exts/filtering/_filter_lists/invite.py | 2 +- bot/exts/filtering/_ui.py | 40 ++++++++++++++----- bot/exts/filtering/filtering.py | 14 ++++--- 5 files changed, 54 insertions(+), 32 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/__init__.py b/bot/exts/filtering/_filter_lists/__init__.py index 1273e5588a..82e0452f98 100644 --- a/bot/exts/filtering/_filter_lists/__init__.py +++ b/bot/exts/filtering/_filter_lists/__init__.py @@ -1,9 +1,9 @@ from os.path import dirname -from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType, ListTypeConverter +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType, list_type_converter from bot.exts.filtering._utils import subclasses_in_package filter_list_types = subclasses_in_package(dirname(__file__), f"{__name__}.", FilterList) filter_list_types = {filter_list.name: filter_list for filter_list in filter_list_types} -__all__ = [filter_list_types, FilterList, ListType, ListTypeConverter] +__all__ = [filter_list_types, FilterList, ListType, list_type_converter] diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 672811444c..5fc9925974 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Dict, List, Optional, Type -from discord.ext.commands import BadArgument, Context, Converter +from discord.ext.commands import BadArgument from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter @@ -20,20 +20,20 @@ class ListType(Enum): ALLOW = 1 -class ListTypeConverter(Converter): - """A Converter to get the appropriate list type.""" +# Alternative names with which each list type can be specified in commands. +aliases = ( + (ListType.DENY, {"deny", "blocklist", "blacklist", "denylist", "bl", "dl"}), + (ListType.ALLOW, {"allow", "allowlist", "whitelist", "al", "wl"}) +) - aliases = ( - (ListType.DENY, {"deny", "blocklist", "blacklist", "denylist", "bl", "dl"}), - (ListType.ALLOW, {"allow", "allowlist", "whitelist", "al", "wl"}) - ) - async def convert(self, ctx: Context, argument: str) -> ListType: - """Get the appropriate list type.""" - for list_type, aliases in self.aliases: - if argument in aliases or argument in map(past_tense, aliases): - return list_type - raise BadArgument(f"No matching list type found for {argument!r}.") +def list_type_converter(argument: str) -> ListType: + """A converter to get the appropriate list type.""" + argument = argument.lower() + for list_type, list_aliases in aliases: + if argument in list_aliases or argument in map(past_tense, list_aliases): + return list_type + raise BadArgument(f"No matching list type found for {argument!r}.") class FilterList(FieldRequiring): diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 04afff0f71..cadd82d0c1 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -5,7 +5,7 @@ from operator import or_ from typing import Optional -from botcore.regex import DISCORD_INVITE +from botcore.utils.regex import DISCORD_INVITE from discord import Embed, Invite from discord.errors import NotFound diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 95a840be87..efedb2c0c8 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -1,10 +1,9 @@ -from copy import copy +from typing import Callable, Optional import discord import discord.ui from discord.ext.commands import Context -import bot from bot.log import get_logger log = get_logger(__name__) @@ -13,30 +12,51 @@ class ArgumentCompletionSelect(discord.ui.Select): """A select detailing the options that can be picked to assign to a missing argument.""" - def __init__(self, ctx: Context, arg_name: str, options: list[str]): + def __init__( + self, + ctx: Context, + args: list, + arg_name: str, + options: list[str], + position: int, + converter: Optional[Callable] = None + ): super().__init__( placeholder=f"Select a value for {arg_name!r}", options=[discord.SelectOption(label=option) for option in options] ) self.ctx = ctx + self.args = args + self.position = position + self.converter = converter async def callback(self, interaction: discord.Interaction) -> None: """re-invoke the context command with the completed argument value.""" await interaction.response.defer() - value = interaction.data['values'][0] - message = copy(self.ctx.message) - message.content = f'{message.content} "{value}"' - log.trace(f"Argument filled with the value {value}. Invoking {message.content!r}") - await bot.instance.process_commands(message) + value = interaction.data["values"][0] + if self.converter: + value = self.converter(value) + args = self.args.copy() # This makes the view reusable. + args.insert(self.position, value) + log.trace(f"Argument filled with the value {value}. Re-invoking command") + await self.ctx.invoke(self.ctx.command, *args) class ArgumentCompletionView(discord.ui.View): """A view used to complete a missing argument in an in invoked command.""" - def __init__(self, ctx: Context, arg_name: str, options: list[str]): + def __init__( + self, + ctx: Context, + args: list, + arg_name: str, + options: list[str], + position: int, + converter: Optional[Callable] = None + ): super().__init__() log.trace(f"The {arg_name} argument was designated missing in the invocation {ctx.view.buffer!r}") - self.add_item(ArgumentCompletionSelect(ctx, arg_name, options)) + self.add_item(ArgumentCompletionSelect(ctx, args, arg_name, options, position, converter)) self.ctx = ctx async def interaction_check(self, interaction: discord.Interaction) -> bool: diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 5eefdf4e47..2e5cca5fae 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -11,7 +11,7 @@ from bot.bot import Bot from bot.constants import Colours, MODERATION_ROLES, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext -from bot.exts.filtering._filter_lists import FilterList, ListType, ListTypeConverter, filter_list_types +from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._ui import ArgumentCompletionView from bot.exts.filtering._utils import past_tense @@ -114,7 +114,7 @@ async def bl_list(self, ctx: Context, list_name: Optional[str] = None) -> None: if list_name is None: await ctx.send( "The **list_name** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, "list_name", list(self.filter_lists)) + view=ArgumentCompletionView(ctx, [], "list_name", list(self.filter_lists), 1, None) ) return await self._send_list(ctx, list_name, ListType.DENY) @@ -134,7 +134,7 @@ async def al_list(self, ctx: Context, list_name: Optional[str] = None) -> None: if list_name is None: await ctx.send( "The **list_name** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, "list_name", list(self.filter_lists)) + view=ArgumentCompletionView(ctx, [], "list_name", list(self.filter_lists), 1, None) ) return await self._send_list(ctx, list_name, ListType.ALLOW) @@ -150,13 +150,13 @@ async def filter(self, ctx: Context) -> None: @filter.command(name="list", aliases=("get",)) async def f_list( - self, ctx: Context, list_type: Optional[ListTypeConverter] = None, list_name: Optional[str] = None + self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None ) -> None: """List the contents of a specified list of filters.""" if list_name is None: await ctx.send( "The **list_name** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, "list_name", list(self.filter_lists)) + view=ArgumentCompletionView(ctx, [list_type], "list_name", list(self.filter_lists), 1, None) ) return @@ -165,7 +165,9 @@ async def f_list( if len(filter_list.filter_lists) > 1: await ctx.send( "The **list_type** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, "list_type", [option.name for option in ListType]) + view=ArgumentCompletionView( + ctx, [list_name], "list_type", [option.name for option in ListType], 0, list_type_converter + ) ) return list_type = list(filter_list.filter_lists)[0] From 1bc334457b36f79fd30f4b85b961c9c096a3c3bc Mon Sep 17 00:00:00 2001 From: mbaruh Date: Wed, 2 Mar 2022 00:08:34 +0200 Subject: [PATCH 008/132] Add domain filtering The domain filtering works very similarly to the token filtering, and the domain matching itself is based on the implementation in the old system. The deletion setting is accessed explicitly in the domain filter in order to allow DMing the user the domain the message got deleted for. This is fine, since practical uses are more important than the theory this system was designed with. Of course, breaking the design should still be avoided whenever possible. --- bot/exts/filtering/_filter_context.py | 1 + bot/exts/filtering/_filter_lists/domain.py | 54 +++++++++++++++++++ bot/exts/filtering/_filters/domain.py | 23 ++++++++ bot/exts/filtering/_settings.py | 22 +++++--- .../_settings_types/delete_messages.py | 7 ++- .../infraction_and_notification.py | 8 ++- 6 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 bot/exts/filtering/_filter_lists/domain.py create mode 100644 bot/exts/filtering/_filters/domain.py diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 2fec9ce428..02738d452d 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -33,6 +33,7 @@ class FilterContext: alert_embeds: list = field(default_factory=list) # Any embeds to add to the alert action_descriptions: list = field(default_factory=list) # What actions were taken matches: list = field(default_factory=list) # What exactly was found + notification_domain: str = field(default_factory=str) # A domain to send the user for context def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py new file mode 100644 index 0000000000..a843283942 --- /dev/null +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import re +import typing +from functools import reduce +from operator import or_ +from typing import Optional + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType +from bot.exts.filtering._filters.domain import DomainFilter +from bot.exts.filtering._settings import ActionSettings +from bot.exts.filtering._utils import clean_input + +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + +URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) + + +class DomainsList(FilterList): + """A list of filters, each looking for a specific domain given by URL.""" + + name = "domain" + + def __init__(self, filtering_cog: Filtering): + super().__init__(DomainFilter) + filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: + """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + text = ctx.content + if not text: + return None, "" + + text = clean_input(text) + urls = {match.group(1).lower() for match in URL_RE.finditer(text)} + new_ctx = ctx.replace(content=urls) + + triggers = self.filter_list_result( + new_ctx, self.filter_lists[ListType.DENY], self.defaults[ListType.DENY]["validations"] + ) + ctx.notification_domain = new_ctx.notification_domain + actions = None + message = "" + if triggers: + actions = reduce(or_, (filter_.actions for filter_ in triggers)) + if len(triggers) == 1: + message = f"#{triggers[0].id} (`{triggers[0].content}`)" + if triggers[0].description: + message += f" - {triggers[0].description}" + else: + message = ", ".join(f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers) + return actions, message diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py new file mode 100644 index 0000000000..5d48c545f0 --- /dev/null +++ b/bot/exts/filtering/_filters/domain.py @@ -0,0 +1,23 @@ +import tldextract + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filters.filter import Filter + + +class DomainFilter(Filter): + """A filter which looks for a specific domain given by URL.""" + + def triggered_on(self, ctx: FilterContext) -> bool: + """Searches for a domain within a given context.""" + domain = tldextract.extract(self.content).registered_domain + + for found_url in ctx.content: + if self.content in found_url and tldextract.extract(found_url).registered_domain == domain: + ctx.matches.append(self.content) + if ( + ("delete_messages" in self.actions and self.actions.get("delete_messages").delete_messages) + or not ctx.notification_domain + ): # Override this field only if this filter causes deletion. + ctx.notification_domain = self.content + return True + return False diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index 96e1c1f7f4..b53400b78c 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -1,6 +1,7 @@ from __future__ import annotations + from abc import abstractmethod -from typing import Iterator, Mapping, Optional +from typing import Any, Iterator, Mapping, Optional, TypeVar from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types import settings_types @@ -8,6 +9,8 @@ from bot.exts.filtering._utils import FieldRequiring from bot.log import get_logger +TSettings = TypeVar("TSettings", bound="Settings") + log = get_logger(__name__) _already_warned: set[str] = set() @@ -15,7 +18,7 @@ def create_settings(settings_data: dict) -> tuple[Optional[ActionSettings], Optional[ValidationSettings]]: """ - Create and return instances of the Settings subclasses from the given data + Create and return instances of the Settings subclasses from the given data. Additionally, warn for data entries with no matching class. """ @@ -75,23 +78,30 @@ def __init__(self, settings_data: dict): f"Attempted to load a {entry_name} setting, but the response is malformed: {entry_data}" ) from e - def __contains__(self, item) -> bool: + def __contains__(self, item: str) -> bool: return item in self._entries def __setitem__(self, key: str, value: entry_type) -> None: self._entries[key] = value - def copy(self): + def copy(self: TSettings) -> TSettings: + """Create a shallow copy of the object.""" copy = self.__class__({}) - copy._entries = self._entries + copy._entries = self._entries.copy() return copy def items(self) -> Iterator[tuple[str, entry_type]]: + """Return an iterator for the items in the entries dictionary.""" yield from self._entries.items() def update(self, mapping: Mapping[str, entry_type], **kwargs: entry_type) -> None: + """Update the entries with items from `mapping` and the kwargs.""" self._entries.update(mapping, **kwargs) + def get(self, key: str, default: Optional[Any] = None) -> entry_type: + """Get the entry matching the key, or fall back to the default value if the key is missing.""" + return self._entries.get(key, default) + @classmethod def create(cls, settings_data: dict) -> Optional[Settings]: """ @@ -152,7 +162,7 @@ def __init__(self, settings_data: dict): super().__init__(settings_data) def __or__(self, other: ActionSettings) -> ActionSettings: - """Combine the entries of two collections of settings into a new ActionsSettings""" + """Combine the entries of two collections of settings into a new ActionsSettings.""" actions = {} # A settings object doesn't necessarily have all types of entries (e.g in the case of filter overrides). for entry in self._entries: diff --git a/bot/exts/filtering/_settings_types/delete_messages.py b/bot/exts/filtering/_settings_types/delete_messages.py index b0a018433d..ad715f04c9 100644 --- a/bot/exts/filtering/_settings_types/delete_messages.py +++ b/bot/exts/filtering/_settings_types/delete_messages.py @@ -14,11 +14,11 @@ class DeleteMessages(ActionEntry): def __init__(self, entry_data: Any): super().__init__(entry_data) - self.delete: bool = entry_data + self.delete_messages: bool = entry_data async def action(self, ctx: FilterContext) -> None: """Delete the context message(s).""" - if not self.delete or ctx.event not in (Event.MESSAGE, Event.MESSAGE_EDIT): + if not self.delete_messages or ctx.event not in (Event.MESSAGE, Event.MESSAGE_EDIT): return with suppress(NotFound): @@ -31,5 +31,4 @@ def __or__(self, other: ActionEntry): if not isinstance(other, DeleteMessages): return NotImplemented - return DeleteMessages(self.delete or other.delete) - + return DeleteMessages(self.delete_messages or other.delete_messages) diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index d308bf444c..82e2ff6d65 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -80,8 +80,12 @@ async def action(self, ctx: FilterContext) -> None: dm_embed = self.dm_embed if dm_content or dm_embed: - dm_content = f"Hey {ctx.author.mention}!\n{dm_content}" - dm_embed = Embed(description=dm_embed, colour=Colour.og_blurple()) if dm_embed else None + formatting = {"domain": ctx.notification_domain} + dm_content = f"Hey {ctx.author.mention}!\n{dm_content.format(**formatting)}" + if dm_embed: + dm_embed = Embed(description=dm_embed.format(**formatting), colour=Colour.og_blurple()) + else: + dm_embed = None try: await ctx.author.send(dm_content, embed=dm_embed) From ddf0839661fa3863b82e6c68dfaa92103d05a252 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 3 Mar 2022 20:21:42 +0200 Subject: [PATCH 009/132] Add filter-type-specific settings Pydantic is added to the bot's dependencies, allowing easily setting up settings that are specific to a filter type (e.g for domains). The data for those settings is loaded from the "additional_field" JSON field in the Filter table. This commit specifically adds an option to match a domain exactly, without matching subdomains. There are several benefits to using Pydantic: - Built-in serialization to, and de-serialization from JSON, which is useful for reading and writing to the DB. - Type validation on initialization. - By default ignores extra fields when de-serializing, which can prevent things breaking if the extra fields change. --- bot/exts/filtering/_filter_lists/domain.py | 4 +- bot/exts/filtering/_filters/domain.py | 17 +- bot/exts/filtering/_filters/filter.py | 2 +- poetry.lock | 968 +++++++++++++++++++-- pyproject.toml | 1 + 5 files changed, 899 insertions(+), 93 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index a843283942..c481643694 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -15,7 +15,7 @@ if typing.TYPE_CHECKING: from bot.exts.filtering.filtering import Filtering -URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) +URL_RE = re.compile(r"https?://([^\s]+)", flags=re.IGNORECASE) class DomainsList(FilterList): @@ -34,7 +34,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings return None, "" text = clean_input(text) - urls = {match.group(1).lower() for match in URL_RE.finditer(text)} + urls = {match.group(1).lower().rstrip("/") for match in URL_RE.finditer(text)} new_ctx = ctx.replace(content=urls) triggers = self.filter_list_result( diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 5d48c545f0..1511d6a5cc 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -1,12 +1,27 @@ +from typing import Optional + import tldextract +from pydantic import BaseModel from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._settings import ActionSettings + + +class ExtraDomainSettings(BaseModel): + """Extra settings for how domains should be matched in a message.""" + + # whether to match the filter content exactly, or to trigger for subdomains and subpaths as well. + exact: Optional[bool] = False class DomainFilter(Filter): """A filter which looks for a specific domain given by URL.""" + def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): + super().__init__(filter_data, action_defaults) + self.extra_fields = ExtraDomainSettings.parse_raw(self.extra_fields) + def triggered_on(self, ctx: FilterContext) -> bool: """Searches for a domain within a given context.""" domain = tldextract.extract(self.content).registered_domain @@ -19,5 +34,5 @@ def triggered_on(self, ctx: FilterContext) -> bool: or not ctx.notification_domain ): # Override this field only if this filter causes deletion. ctx.notification_domain = self.content - return True + return not self.extra_fields.exact or self.content == found_url return False diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index f1e5eac916..b4beb8386d 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -22,7 +22,7 @@ def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] self.actions = action_defaults elif action_defaults: self.actions.fallback_to(action_defaults) - self.exact = filter_data["additional_field"] + self.extra_fields = filter_data["additional_field"] or "{}" # noqa: P103 @abstractmethod def triggered_on(self, ctx: FilterContext) -> bool: diff --git a/poetry.lock b/poetry.lock index f41fe2358a..01d0812596 100644 --- a/poetry.lock +++ b/poetry.lock @@ -88,7 +88,7 @@ python-versions = ">=3.6" [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "dev" optional = false @@ -142,7 +142,6 @@ async-rediscache = ["async-rediscache[fakeredis] (==0.2.0)"] [package.source] type = "url" url = "https://github.com/python-discord/bot-core/archive/refs/tags/v7.2.0.zip" - [[package]] name = "certifi" version = "2022.6.15" @@ -153,7 +152,7 @@ python-versions = ">=3.6" [[package]] name = "cffi" -version = "1.15.0" +version = "1.15.1" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -268,7 +267,7 @@ url = "https://github.com/Rapptz/discord.py/archive/0eb3d26343969a25ffc43ba72eca [[package]] name = "distlib" -version = "0.3.4" +version = "0.3.5" description = "Distribution utilities" category = "dev" optional = false @@ -490,11 +489,11 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "2.7" +version = "3.3" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.5" [[package]] name = "iniconfig" @@ -520,7 +519,7 @@ plugins = ["setuptools"] [[package]] name = "jarowinkler" -version = "1.0.4" +version = "1.0.5" description = "library for fast approximate string matching using Jaro and Jaro-Winkler similarity" category = "main" optional = false @@ -742,6 +741,21 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pydantic" +version = "1.9.1" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pydocstyle" version = "6.1.1" @@ -1098,22 +1112,30 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "urllib3" -version = "1.26.5" +version = "1.26.10" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] -brotli = ["brotlipy (>=0.6.0)"] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.15.0" +version = "20.15.1" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -1152,104 +1174,872 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "36098ac3587aada7726bd4cb8f8c9a450d4aa07f849daefebf896d11d020af7d" +content-hash = "884620193214be6da308b15308c77a9951916ab8b4fc2c5d3071488e7eacf36b" [metadata.files] -aiodns = [] -aiohttp = [] -aioredis = [] -aiosignal = [] -arrow = [] -async-rediscache = [] -async-timeout = [] +aiodns = [ + {file = "aiodns-3.0.0-py3-none-any.whl", hash = "sha256:2b19bc5f97e5c936638d28e665923c093d8af2bf3aa88d35c43417fa25d136a2"}, + {file = "aiodns-3.0.0.tar.gz", hash = "sha256:946bdfabe743fceeeb093c8a010f5d1645f708a241be849e17edfb0e49e08cd6"}, +] +aiohttp = [ + {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1ed0b6477896559f17b9eaeb6d38e07f7f9ffe40b9f0f9627ae8b9926ae260a8"}, + {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7dadf3c307b31e0e61689cbf9e06be7a867c563d5a63ce9dca578f956609abf8"}, + {file = "aiohttp-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a79004bb58748f31ae1cbe9fa891054baaa46fb106c2dc7af9f8e3304dc30316"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12de6add4038df8f72fac606dff775791a60f113a725c960f2bab01d8b8e6b15"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f0d5f33feb5f69ddd57a4a4bd3d56c719a141080b445cbf18f238973c5c9923"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eaba923151d9deea315be1f3e2b31cc39a6d1d2f682f942905951f4e40200922"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:099ebd2c37ac74cce10a3527d2b49af80243e2a4fa39e7bce41617fbc35fa3c1"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e5d962cf7e1d426aa0e528a7e198658cdc8aa4fe87f781d039ad75dcd52c516"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61bfc23df345d8c9716d03717c2ed5e27374e0fe6f659ea64edcd27b4b044cf7"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:31560d268ff62143e92423ef183680b9829b1b482c011713ae941997921eebc8"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:01d7bdb774a9acc838e6b8f1d114f45303841b89b95984cbb7d80ea41172a9e3"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97ef77eb6b044134c0b3a96e16abcb05ecce892965a2124c566af0fd60f717e2"}, + {file = "aiohttp-3.8.1-cp310-cp310-win32.whl", hash = "sha256:c2aef4703f1f2ddc6df17519885dbfa3514929149d3ff900b73f45998f2532fa"}, + {file = "aiohttp-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:713ac174a629d39b7c6a3aa757b337599798da4c1157114a314e4e391cd28e32"}, + {file = "aiohttp-3.8.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:473d93d4450880fe278696549f2e7aed8cd23708c3c1997981464475f32137db"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b5eeae8e019e7aad8af8bb314fb908dd2e028b3cdaad87ec05095394cce632"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af642b43ce56c24d063325dd2cf20ee012d2b9ba4c3c008755a301aaea720ad"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3630c3ef435c0a7c549ba170a0633a56e92629aeed0e707fec832dee313fb7a"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a4a4e30bf1edcad13fb0804300557aedd07a92cabc74382fdd0ba6ca2661091"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6f8b01295e26c68b3a1b90efb7a89029110d3a4139270b24fda961893216c440"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a25fa703a527158aaf10dafd956f7d42ac6d30ec80e9a70846253dd13e2f067b"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:5bfde62d1d2641a1f5173b8c8c2d96ceb4854f54a44c23102e2ccc7e02f003ec"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:51467000f3647d519272392f484126aa716f747859794ac9924a7aafa86cd411"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:03a6d5349c9ee8f79ab3ff3694d6ce1cfc3ced1c9d36200cb8f08ba06bd3b782"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:102e487eeb82afac440581e5d7f8f44560b36cf0bdd11abc51a46c1cd88914d4"}, + {file = "aiohttp-3.8.1-cp36-cp36m-win32.whl", hash = "sha256:4aed991a28ea3ce320dc8ce655875e1e00a11bdd29fe9444dd4f88c30d558602"}, + {file = "aiohttp-3.8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b0e20cddbd676ab8a64c774fefa0ad787cc506afd844de95da56060348021e96"}, + {file = "aiohttp-3.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:37951ad2f4a6df6506750a23f7cbabad24c73c65f23f72e95897bb2cecbae676"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c23b1ad869653bc818e972b7a3a79852d0e494e9ab7e1a701a3decc49c20d51"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15b09b06dae900777833fe7fc4b4aa426556ce95847a3e8d7548e2d19e34edb8"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:477c3ea0ba410b2b56b7efb072c36fa91b1e6fc331761798fa3f28bb224830dd"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2f2f69dca064926e79997f45b2f34e202b320fd3782f17a91941f7eb85502ee2"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ef9612483cb35171d51d9173647eed5d0069eaa2ee812793a75373447d487aa4"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6d69f36d445c45cda7b3b26afef2fc34ef5ac0cdc75584a87ef307ee3c8c6d00"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:55c3d1072704d27401c92339144d199d9de7b52627f724a949fc7d5fc56d8b93"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d00268fcb9f66fbcc7cd9fe423741d90c75ee029a1d15c09b22d23253c0a44"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:07b05cd3305e8a73112103c834e91cd27ce5b4bd07850c4b4dbd1877d3f45be7"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c34dc4958b232ef6188c4318cb7b2c2d80521c9a56c52449f8f93ab7bc2a8a1c"}, + {file = "aiohttp-3.8.1-cp37-cp37m-win32.whl", hash = "sha256:d2f9b69293c33aaa53d923032fe227feac867f81682f002ce33ffae978f0a9a9"}, + {file = "aiohttp-3.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6ae828d3a003f03ae31915c31fa684b9890ea44c9c989056fea96e3d12a9fa17"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0c7ebbbde809ff4e970824b2b6cb7e4222be6b95a296e46c03cf050878fc1785"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b7ef7cbd4fec9a1e811a5de813311ed4f7ac7d93e0fda233c9b3e1428f7dd7b"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c3d6a4d0619e09dcd61021debf7059955c2004fa29f48788a3dfaf9c9901a7cd"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:718626a174e7e467f0558954f94af117b7d4695d48eb980146016afa4b580b2e"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:589c72667a5febd36f1315aa6e5f56dd4aa4862df295cb51c769d16142ddd7cd"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ed076098b171573161eb146afcb9129b5ff63308960aeca4b676d9d3c35e700"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:086f92daf51a032d062ec5f58af5ca6a44d082c35299c96376a41cbb33034675"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:11691cf4dc5b94236ccc609b70fec991234e7ef8d4c02dd0c9668d1e486f5abf"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:31d1e1c0dbf19ebccbfd62eff461518dcb1e307b195e93bba60c965a4dcf1ba0"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:11a67c0d562e07067c4e86bffc1553f2cf5b664d6111c894671b2b8712f3aba5"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:bb01ba6b0d3f6c68b89fce7305080145d4877ad3acaed424bae4d4ee75faa950"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:44db35a9e15d6fe5c40d74952e803b1d96e964f683b5a78c3cc64eb177878155"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:844a9b460871ee0a0b0b68a64890dae9c415e513db0f4a7e3cab41a0f2fedf33"}, + {file = "aiohttp-3.8.1-cp38-cp38-win32.whl", hash = "sha256:7d08744e9bae2ca9c382581f7dce1273fe3c9bae94ff572c3626e8da5b193c6a"}, + {file = "aiohttp-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:04d48b8ce6ab3cf2097b1855e1505181bdd05586ca275f2505514a6e274e8e75"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f5315a2eb0239185af1bddb1abf472d877fede3cc8d143c6cddad37678293237"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a996d01ca39b8dfe77440f3cd600825d05841088fd6bc0144cc6c2ec14cc5f74"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:13487abd2f761d4be7c8ff9080de2671e53fff69711d46de703c310c4c9317ca"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea302f34477fda3f85560a06d9ebdc7fa41e82420e892fc50b577e35fc6a50b2"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f635ce61a89c5732537a7896b6319a8fcfa23ba09bec36e1b1ac0ab31270d2"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e999f2d0e12eea01caeecb17b653f3713d758f6dcc770417cf29ef08d3931421"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0770e2806a30e744b4e21c9d73b7bee18a1cfa3c47991ee2e5a65b887c49d5cf"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d15367ce87c8e9e09b0f989bfd72dc641bcd04ba091c68cd305312d00962addd"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c7cefb4b0640703eb1069835c02486669312bf2f12b48a748e0a7756d0de33d"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:71927042ed6365a09a98a6377501af5c9f0a4d38083652bcd2281a06a5976724"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:28d490af82bc6b7ce53ff31337a18a10498303fe66f701ab65ef27e143c3b0ef"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:b6613280ccedf24354406caf785db748bebbddcf31408b20c0b48cb86af76866"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81e3d8c34c623ca4e36c46524a3530e99c0bc95ed068fd6e9b55cb721d408fb2"}, + {file = "aiohttp-3.8.1-cp39-cp39-win32.whl", hash = "sha256:7187a76598bdb895af0adbd2fb7474d7f6025d170bc0a1130242da817ce9e7d1"}, + {file = "aiohttp-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:1c182cb873bc91b411e184dab7a2b664d4fea2743df0e4d57402f7f3fa644bac"}, + {file = "aiohttp-3.8.1.tar.gz", hash = "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578"}, +] +aioredis = [ + {file = "aioredis-1.3.1-py3-none-any.whl", hash = "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"}, + {file = "aioredis-1.3.1.tar.gz", hash = "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a"}, +] +aiosignal = [ + {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"}, + {file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"}, +] +arrow = [ + {file = "arrow-1.2.2-py3-none-any.whl", hash = "sha256:d622c46ca681b5b3e3574fcb60a04e5cc81b9625112d5fb2b44220c36c892177"}, + {file = "arrow-1.2.2.tar.gz", hash = "sha256:05caf1fd3d9a11a1135b2b6f09887421153b94558e5ef4d090b567b47173ac2b"}, +] +async-rediscache = [ + {file = "async-rediscache-0.2.0.tar.gz", hash = "sha256:c1fd95fe530211b999748ebff96e2e9b629f2664957f9b36916b898e42fc57c4"}, + {file = "async_rediscache-0.2.0-py3-none-any.whl", hash = "sha256:710676211b407399c9ad94afa66fa04c22a936be11ba6f227e6c74cfa140ce78"}, +] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] atomicwrites = [] -attrs = [] -beautifulsoup4 = [] +attrs = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.10.0-py3-none-any.whl", hash = "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf"}, + {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"}, +] bot-core = [] certifi = [] cffi = [] -cfgv = [] +cfgv = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] charset-normalizer = [] -colorama = [] -coloredlogs = [] -coverage = [] -deepdiff = [] -deprecated = [] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +coloredlogs = [ + {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, + {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, +] +coverage = [ + {file = "coverage-6.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b27d894748475fa858f9597c0ee1d4829f44683f3813633aaf94b19cb5453cf"}, + {file = "coverage-6.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37d1141ad6b2466a7b53a22e08fe76994c2d35a5b6b469590424a9953155afac"}, + {file = "coverage-6.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9987b0354b06d4df0f4d3e0ec1ae76d7ce7cbca9a2f98c25041eb79eec766f1"}, + {file = "coverage-6.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26e2deacd414fc2f97dd9f7676ee3eaecd299ca751412d89f40bc01557a6b1b4"}, + {file = "coverage-6.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dd8bafa458b5c7d061540f1ee9f18025a68e2d8471b3e858a9dad47c8d41903"}, + {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:46191097ebc381fbf89bdce207a6c107ac4ec0890d8d20f3360345ff5976155c"}, + {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6f89d05e028d274ce4fa1a86887b071ae1755082ef94a6740238cd7a8178804f"}, + {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:58303469e9a272b4abdb9e302a780072c0633cdcc0165db7eec0f9e32f901e05"}, + {file = "coverage-6.3.2-cp310-cp310-win32.whl", hash = "sha256:2fea046bfb455510e05be95e879f0e768d45c10c11509e20e06d8fcaa31d9e39"}, + {file = "coverage-6.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:a2a8b8bcc399edb4347a5ca8b9b87e7524c0967b335fbb08a83c8421489ddee1"}, + {file = "coverage-6.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f1555ea6d6da108e1999b2463ea1003fe03f29213e459145e70edbaf3e004aaa"}, + {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5f4e1edcf57ce94e5475fe09e5afa3e3145081318e5fd1a43a6b4539a97e518"}, + {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a15dc0a14008f1da3d1ebd44bdda3e357dbabdf5a0b5034d38fcde0b5c234b7"}, + {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b7745788866028adeb1e0eca3bf1101109e2dc58456cb49d2d9b99a8c516e6"}, + {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8ce257cac556cb03be4a248d92ed36904a59a4a5ff55a994e92214cde15c5bad"}, + {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b0be84e5a6209858a1d3e8d1806c46214e867ce1b0fd32e4ea03f4bd8b2e3359"}, + {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:acf53bc2cf7282ab9b8ba346746afe703474004d9e566ad164c91a7a59f188a4"}, + {file = "coverage-6.3.2-cp37-cp37m-win32.whl", hash = "sha256:8bdde1177f2311ee552f47ae6e5aa7750c0e3291ca6b75f71f7ffe1f1dab3dca"}, + {file = "coverage-6.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b31651d018b23ec463e95cf10070d0b2c548aa950a03d0b559eaa11c7e5a6fa3"}, + {file = "coverage-6.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d"}, + {file = "coverage-6.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c6dbb42f3ad25760010c45191e9757e7dce981cbfb90e42feef301d71540059"}, + {file = "coverage-6.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c76aeef1b95aff3905fb2ae2d96e319caca5b76fa41d3470b19d4e4a3a313512"}, + {file = "coverage-6.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cf5cfcb1521dc3255d845d9dca3ff204b3229401994ef8d1984b32746bb45ca"}, + {file = "coverage-6.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fbbdc8d55990eac1b0919ca69eb5a988a802b854488c34b8f37f3e2025fa90d"}, + {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ec6bc7fe73a938933d4178c9b23c4e0568e43e220aef9472c4f6044bfc6dd0f0"}, + {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9baff2a45ae1f17c8078452e9e5962e518eab705e50a0aa8083733ea7d45f3a6"}, + {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd9e830e9d8d89b20ab1e5af09b32d33e1a08ef4c4e14411e559556fd788e6b2"}, + {file = "coverage-6.3.2-cp38-cp38-win32.whl", hash = "sha256:f7331dbf301b7289013175087636bbaf5b2405e57259dd2c42fdcc9fcc47325e"}, + {file = "coverage-6.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:68353fe7cdf91f109fc7d474461b46e7f1f14e533e911a2a2cbb8b0fc8613cf1"}, + {file = "coverage-6.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b78e5afb39941572209f71866aa0b206c12f0109835aa0d601e41552f9b3e620"}, + {file = "coverage-6.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4e21876082ed887baed0146fe222f861b5815455ada3b33b890f4105d806128d"}, + {file = "coverage-6.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34626a7eee2a3da12af0507780bb51eb52dca0e1751fd1471d0810539cefb536"}, + {file = "coverage-6.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ebf730d2381158ecf3dfd4453fbca0613e16eaa547b4170e2450c9707665ce7"}, + {file = "coverage-6.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd6fe30bd519694b356cbfcaca9bd5c1737cddd20778c6a581ae20dc8c04def2"}, + {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:96f8a1cb43ca1422f36492bebe63312d396491a9165ed3b9231e778d43a7fca4"}, + {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dd035edafefee4d573140a76fdc785dc38829fe5a455c4bb12bac8c20cfc3d69"}, + {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5ca5aeb4344b30d0bec47481536b8ba1181d50dbe783b0e4ad03c95dc1296684"}, + {file = "coverage-6.3.2-cp39-cp39-win32.whl", hash = "sha256:f5fa5803f47e095d7ad8443d28b01d48c0359484fec1b9d8606d0e3282084bc4"}, + {file = "coverage-6.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:9548f10d8be799551eb3a9c74bbf2b4934ddb330e08a73320123c07f95cc2d92"}, + {file = "coverage-6.3.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:18d520c6860515a771708937d2f78f63cc47ab3b80cb78e86573b0a760161faf"}, + {file = "coverage-6.3.2.tar.gz", hash = "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9"}, +] +deepdiff = [ + {file = "deepdiff-5.7.0-py3-none-any.whl", hash = "sha256:1ffb38c3b5d9174eb2df95850c93aee55ec00e19396925036a2e680f725079e0"}, + {file = "deepdiff-5.7.0.tar.gz", hash = "sha256:838766484e323dcd9dec6955926a893a83767dc3f3f94542773e6aa096efe5d4"}, +] +deprecated = [ + {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, + {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, +] "discord.py" = [] distlib = [] -emoji = [] -execnet = [] +emoji = [ + {file = "emoji-1.7.0.tar.gz", hash = "sha256:65c54533ea3c78f30d0729288998715f418d7467de89ec258a31c0ce8660a1d1"}, +] +execnet = [ + {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, + {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, +] fakeredis = [] -feedparser = [] +feedparser = [ + {file = "feedparser-6.0.8-py3-none-any.whl", hash = "sha256:1b7f57841d9cf85074deb316ed2c795091a238adb79846bc46dccdaf80f9c59a"}, + {file = "feedparser-6.0.8.tar.gz", hash = "sha256:5ce0410a05ab248c8c7cfca3a0ea2203968ee9ff4486067379af4827a59f9661"}, +] filelock = [] -flake8 = [] -flake8-annotations = [] -flake8-bugbear = [] -flake8-docstrings = [] -flake8-isort = [] -flake8-polyfill = [] -flake8-string-format = [] -flake8-tidy-imports = [] -flake8-todo = [] -frozenlist = [] -hiredis = [] -humanfriendly = [] +flake8 = [ + {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, + {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, +] +flake8-annotations = [ + {file = "flake8-annotations-2.8.0.tar.gz", hash = "sha256:a2765c6043098aab0a3f519b871b33586c7fba7037686404b920cf8100cc1cdc"}, + {file = "flake8_annotations-2.8.0-py3-none-any.whl", hash = "sha256:880f9bb0677b82655f9021112d64513e03caefd2e0d786ab4a59ddb5b262caa9"}, +] +flake8-bugbear = [ + {file = "flake8-bugbear-22.3.23.tar.gz", hash = "sha256:e0dc2a36474490d5b1a2d57f9e4ef570abc09f07cbb712b29802e28a2367ff19"}, + {file = "flake8_bugbear-22.3.23-py3-none-any.whl", hash = "sha256:ec5ec92195720cee1589315416b844ffa5e82f73a78e65329e8055322df1e939"}, +] +flake8-docstrings = [ + {file = "flake8-docstrings-1.6.0.tar.gz", hash = "sha256:9fe7c6a306064af8e62a055c2f61e9eb1da55f84bb39caef2b84ce53708ac34b"}, + {file = "flake8_docstrings-1.6.0-py2.py3-none-any.whl", hash = "sha256:99cac583d6c7e32dd28bbfbef120a7c0d1b6dde4adb5a9fd441c4227a6534bde"}, +] +flake8-isort = [ + {file = "flake8-isort-4.1.1.tar.gz", hash = "sha256:d814304ab70e6e58859bc5c3e221e2e6e71c958e7005239202fee19c24f82717"}, + {file = "flake8_isort-4.1.1-py3-none-any.whl", hash = "sha256:c4e8b6dcb7be9b71a02e6e5d4196cefcef0f3447be51e82730fb336fff164949"}, +] +flake8-polyfill = [ + {file = "flake8-polyfill-1.0.2.tar.gz", hash = "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"}, + {file = "flake8_polyfill-1.0.2-py2.py3-none-any.whl", hash = "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9"}, +] +flake8-string-format = [ + {file = "flake8-string-format-0.3.0.tar.gz", hash = "sha256:65f3da786a1461ef77fca3780b314edb2853c377f2e35069723348c8917deaa2"}, + {file = "flake8_string_format-0.3.0-py2.py3-none-any.whl", hash = "sha256:812ff431f10576a74c89be4e85b8e075a705be39bc40c4b4278b5b13e2afa9af"}, +] +flake8-tidy-imports = [ + {file = "flake8-tidy-imports-4.6.0.tar.gz", hash = "sha256:3e193d8c4bb4492408a90e956d888b27eed14c698387c9b38230da3dad78058f"}, + {file = "flake8_tidy_imports-4.6.0-py3-none-any.whl", hash = "sha256:6ae9f55d628156e19d19f4c359dd5d3e95431a9bd514f5e2748c53c1398c66b2"}, +] +flake8-todo = [ + {file = "flake8-todo-0.7.tar.gz", hash = "sha256:6e4c5491ff838c06fe5a771b0e95ee15fc005ca57196011011280fc834a85915"}, +] +frozenlist = [ + {file = "frozenlist-1.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d2257aaba9660f78c7b1d8fea963b68f3feffb1a9d5d05a18401ca9eb3e8d0a3"}, + {file = "frozenlist-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4a44ebbf601d7bac77976d429e9bdb5a4614f9f4027777f9e54fd765196e9d3b"}, + {file = "frozenlist-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:45334234ec30fc4ea677f43171b18a27505bfb2dba9aca4398a62692c0ea8868"}, + {file = "frozenlist-1.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47be22dc27ed933d55ee55845d34a3e4e9f6fee93039e7f8ebadb0c2f60d403f"}, + {file = "frozenlist-1.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03a7dd1bfce30216a3f51a84e6dd0e4a573d23ca50f0346634916ff105ba6e6b"}, + {file = "frozenlist-1.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:691ddf6dc50480ce49f68441f1d16a4c3325887453837036e0fb94736eae1e58"}, + {file = "frozenlist-1.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bde99812f237f79eaf3f04ebffd74f6718bbd216101b35ac7955c2d47c17da02"}, + {file = "frozenlist-1.3.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a202458d1298ced3768f5a7d44301e7c86defac162ace0ab7434c2e961166e8"}, + {file = "frozenlist-1.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9e3e9e365991f8cc5f5edc1fd65b58b41d0514a6a7ad95ef5c7f34eb49b3d3e"}, + {file = "frozenlist-1.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:04cb491c4b1c051734d41ea2552fde292f5f3a9c911363f74f39c23659c4af78"}, + {file = "frozenlist-1.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:436496321dad302b8b27ca955364a439ed1f0999311c393dccb243e451ff66aa"}, + {file = "frozenlist-1.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:754728d65f1acc61e0f4df784456106e35afb7bf39cfe37227ab00436fb38676"}, + {file = "frozenlist-1.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6eb275c6385dd72594758cbe96c07cdb9bd6becf84235f4a594bdf21e3596c9d"}, + {file = "frozenlist-1.3.0-cp310-cp310-win32.whl", hash = "sha256:e30b2f9683812eb30cf3f0a8e9f79f8d590a7999f731cf39f9105a7c4a39489d"}, + {file = "frozenlist-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:f7353ba3367473d1d616ee727945f439e027f0bb16ac1a750219a8344d1d5d3c"}, + {file = "frozenlist-1.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88aafd445a233dbbf8a65a62bc3249a0acd0d81ab18f6feb461cc5a938610d24"}, + {file = "frozenlist-1.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4406cfabef8f07b3b3af0f50f70938ec06d9f0fc26cbdeaab431cbc3ca3caeaa"}, + {file = "frozenlist-1.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8cf829bd2e2956066dd4de43fd8ec881d87842a06708c035b37ef632930505a2"}, + {file = "frozenlist-1.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:603b9091bd70fae7be28bdb8aa5c9990f4241aa33abb673390a7f7329296695f"}, + {file = "frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25af28b560e0c76fa41f550eacb389905633e7ac02d6eb3c09017fa1c8cdfde1"}, + {file = "frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c7a8a9fc9383b52c410a2ec952521906d355d18fccc927fca52ab575ee8b93"}, + {file = "frozenlist-1.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:65bc6e2fece04e2145ab6e3c47428d1bbc05aede61ae365b2c1bddd94906e478"}, + {file = "frozenlist-1.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3f7c935c7b58b0d78c0beea0c7358e165f95f1fd8a7e98baa40d22a05b4a8141"}, + {file = "frozenlist-1.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd89acd1b8bb4f31b47072615d72e7f53a948d302b7c1d1455e42622de180eae"}, + {file = "frozenlist-1.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:6983a31698490825171be44ffbafeaa930ddf590d3f051e397143a5045513b01"}, + {file = "frozenlist-1.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:adac9700675cf99e3615eb6a0eb5e9f5a4143c7d42c05cea2e7f71c27a3d0846"}, + {file = "frozenlist-1.3.0-cp37-cp37m-win32.whl", hash = "sha256:0c36e78b9509e97042ef869c0e1e6ef6429e55817c12d78245eb915e1cca7468"}, + {file = "frozenlist-1.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:57f4d3f03a18facacb2a6bcd21bccd011e3b75d463dc49f838fd699d074fabd1"}, + {file = "frozenlist-1.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8c905a5186d77111f02144fab5b849ab524f1e876a1e75205cd1386a9be4b00a"}, + {file = "frozenlist-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b5009062d78a8c6890d50b4e53b0ddda31841b3935c1937e2ed8c1bda1c7fb9d"}, + {file = "frozenlist-1.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2fdc3cd845e5a1f71a0c3518528bfdbfe2efaf9886d6f49eacc5ee4fd9a10953"}, + {file = "frozenlist-1.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e650bd09b5dda929523b9f8e7f99b24deac61240ecc1a32aeba487afcd970f"}, + {file = "frozenlist-1.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:40dff8962b8eba91fd3848d857203f0bd704b5f1fa2b3fc9af64901a190bba08"}, + {file = "frozenlist-1.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:768efd082074bb203c934e83a61654ed4931ef02412c2fbdecea0cff7ecd0274"}, + {file = "frozenlist-1.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:006d3595e7d4108a12025ddf415ae0f6c9e736e726a5db0183326fd191b14c5e"}, + {file = "frozenlist-1.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:871d42623ae15eb0b0e9df65baeee6976b2e161d0ba93155411d58ff27483ad8"}, + {file = "frozenlist-1.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aff388be97ef2677ae185e72dc500d19ecaf31b698986800d3fc4f399a5e30a5"}, + {file = "frozenlist-1.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9f892d6a94ec5c7b785e548e42722e6f3a52f5f32a8461e82ac3e67a3bd073f1"}, + {file = "frozenlist-1.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:e982878792c971cbd60ee510c4ee5bf089a8246226dea1f2138aa0bb67aff148"}, + {file = "frozenlist-1.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c6c321dd013e8fc20735b92cb4892c115f5cdb82c817b1e5b07f6b95d952b2f0"}, + {file = "frozenlist-1.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:30530930410855c451bea83f7b272fb1c495ed9d5cc72895ac29e91279401db3"}, + {file = "frozenlist-1.3.0-cp38-cp38-win32.whl", hash = "sha256:40ec383bc194accba825fbb7d0ef3dda5736ceab2375462f1d8672d9f6b68d07"}, + {file = "frozenlist-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:f20baa05eaa2bcd5404c445ec51aed1c268d62600362dc6cfe04fae34a424bd9"}, + {file = "frozenlist-1.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0437fe763fb5d4adad1756050cbf855bbb2bf0d9385c7bb13d7a10b0dd550486"}, + {file = "frozenlist-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b684c68077b84522b5c7eafc1dc735bfa5b341fb011d5552ebe0968e22ed641c"}, + {file = "frozenlist-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93641a51f89473837333b2f8100f3f89795295b858cd4c7d4a1f18e299dc0a4f"}, + {file = "frozenlist-1.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6d32ff213aef0fd0bcf803bffe15cfa2d4fde237d1d4838e62aec242a8362fa"}, + {file = "frozenlist-1.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31977f84828b5bb856ca1eb07bf7e3a34f33a5cddce981d880240ba06639b94d"}, + {file = "frozenlist-1.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c62964192a1c0c30b49f403495911298810bada64e4f03249ca35a33ca0417a"}, + {file = "frozenlist-1.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4eda49bea3602812518765810af732229b4291d2695ed24a0a20e098c45a707b"}, + {file = "frozenlist-1.3.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acb267b09a509c1df5a4ca04140da96016f40d2ed183cdc356d237286c971b51"}, + {file = "frozenlist-1.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e1e26ac0a253a2907d654a37e390904426d5ae5483150ce3adedb35c8c06614a"}, + {file = "frozenlist-1.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f96293d6f982c58ebebb428c50163d010c2f05de0cde99fd681bfdc18d4b2dc2"}, + {file = "frozenlist-1.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e84cb61b0ac40a0c3e0e8b79c575161c5300d1d89e13c0e02f76193982f066ed"}, + {file = "frozenlist-1.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:ff9310f05b9d9c5c4dd472983dc956901ee6cb2c3ec1ab116ecdde25f3ce4951"}, + {file = "frozenlist-1.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d26b650b71fdc88065b7a21f8ace70175bcf3b5bdba5ea22df4bfd893e795a3b"}, + {file = "frozenlist-1.3.0-cp39-cp39-win32.whl", hash = "sha256:01a73627448b1f2145bddb6e6c2259988bb8aee0fb361776ff8604b99616cd08"}, + {file = "frozenlist-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:772965f773757a6026dea111a15e6e2678fbd6216180f82a48a40b27de1ee2ab"}, + {file = "frozenlist-1.3.0.tar.gz", hash = "sha256:ce6f2ba0edb7b0c1d8976565298ad2deba6f8064d2bebb6ffce2ca896eb35b0b"}, +] +hiredis = [ + {file = "hiredis-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b4c8b0bc5841e578d5fb32a16e0c305359b987b850a06964bd5a62739d688048"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0adea425b764a08270820531ec2218d0508f8ae15a448568109ffcae050fee26"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3d55e36715ff06cdc0ab62f9591607c4324297b6b6ce5b58cb9928b3defe30ea"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:5d2a48c80cf5a338d58aae3c16872f4d452345e18350143b3bf7216d33ba7b99"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:240ce6dc19835971f38caf94b5738092cb1e641f8150a9ef9251b7825506cb05"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:5dc7a94bb11096bc4bffd41a3c4f2b958257085c01522aa81140c68b8bf1630a"}, + {file = "hiredis-2.0.0-cp36-cp36m-win32.whl", hash = "sha256:139705ce59d94eef2ceae9fd2ad58710b02aee91e7fa0ccb485665ca0ecbec63"}, + {file = "hiredis-2.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c39c46d9e44447181cd502a35aad2bb178dbf1b1f86cf4db639d7b9614f837c6"}, + {file = "hiredis-2.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:adf4dd19d8875ac147bf926c727215a0faf21490b22c053db464e0bf0deb0485"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0f41827028901814c709e744060843c77e78a3aca1e0d6875d2562372fcb405a"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:508999bec4422e646b05c95c598b64bdbef1edf0d2b715450a078ba21b385bcc"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:0d5109337e1db373a892fdcf78eb145ffb6bbd66bb51989ec36117b9f7f9b579"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:04026461eae67fdefa1949b7332e488224eac9e8f2b5c58c98b54d29af22093e"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a00514362df15af041cc06e97aebabf2895e0a7c42c83c21894be12b84402d79"}, + {file = "hiredis-2.0.0-cp37-cp37m-win32.whl", hash = "sha256:09004096e953d7ebd508cded79f6b21e05dff5d7361771f59269425108e703bc"}, + {file = "hiredis-2.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f8196f739092a78e4f6b1b2172679ed3343c39c61a3e9d722ce6fcf1dac2824a"}, + {file = "hiredis-2.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:294a6697dfa41a8cba4c365dd3715abc54d29a86a40ec6405d677ca853307cfb"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3dddf681284fe16d047d3ad37415b2e9ccdc6c8986c8062dbe51ab9a358b50a5"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:dcef843f8de4e2ff5e35e96ec2a4abbdf403bd0f732ead127bd27e51f38ac298"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:87c7c10d186f1743a8fd6a971ab6525d60abd5d5d200f31e073cd5e94d7e7a9d"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:7f0055f1809b911ab347a25d786deff5e10e9cf083c3c3fd2dd04e8612e8d9db"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:11d119507bb54e81f375e638225a2c057dda748f2b1deef05c2b1a5d42686048"}, + {file = "hiredis-2.0.0-cp38-cp38-win32.whl", hash = "sha256:7492af15f71f75ee93d2a618ca53fea8be85e7b625e323315169977fae752426"}, + {file = "hiredis-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:65d653df249a2f95673976e4e9dd7ce10de61cfc6e64fa7eeaa6891a9559c581"}, + {file = "hiredis-2.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae8427a5e9062ba66fc2c62fb19a72276cf12c780e8db2b0956ea909c48acff5"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:3f5f7e3a4ab824e3de1e1700f05ad76ee465f5f11f5db61c4b297ec29e692b2e"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e3447d9e074abf0e3cd85aef8131e01ab93f9f0e86654db7ac8a3f73c63706ce"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:8b42c0dc927b8d7c0eb59f97e6e34408e53bc489f9f90e66e568f329bff3e443"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:b84f29971f0ad4adaee391c6364e6f780d5aae7e9226d41964b26b49376071d0"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0b39ec237459922c6544d071cdcf92cbb5bc6685a30e7c6d985d8a3e3a75326e"}, + {file = "hiredis-2.0.0-cp39-cp39-win32.whl", hash = "sha256:a7928283143a401e72a4fad43ecc85b35c27ae699cf5d54d39e1e72d97460e1d"}, + {file = "hiredis-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:a4ee8000454ad4486fb9f28b0cab7fa1cd796fc36d639882d0b34109b5b3aec9"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1f03d4dadd595f7a69a75709bc81902673fa31964c75f93af74feac2f134cc54"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-manylinux1_x86_64.whl", hash = "sha256:04927a4c651a0e9ec11c68e4427d917e44ff101f761cd3b5bc76f86aaa431d27"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:a39efc3ade8c1fb27c097fd112baf09d7fd70b8cb10ef1de4da6efbe066d381d"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:07bbf9bdcb82239f319b1f09e8ef4bdfaec50ed7d7ea51a56438f39193271163"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:807b3096205c7cec861c8803a6738e33ed86c9aae76cac0e19454245a6bbbc0a"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-manylinux1_x86_64.whl", hash = "sha256:1233e303645f468e399ec906b6b48ab7cd8391aae2d08daadbb5cad6ace4bd87"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:cb2126603091902767d96bcb74093bd8b14982f41809f85c9b96e519c7e1dc41"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-win32.whl", hash = "sha256:f52010e0a44e3d8530437e7da38d11fb822acfb0d5b12e9cd5ba655509937ca0"}, + {file = "hiredis-2.0.0.tar.gz", hash = "sha256:81d6d8e39695f2c37954d1011c0480ef7cf444d4e3ae24bc5e89ee5de360139a"}, +] +humanfriendly = [ + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, +] identify = [] -idna = [] -iniconfig = [] -isort = [] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +isort = [ + {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, + {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, +] jarowinkler = [] -lupa = [] +lupa = [ + {file = "lupa-1.13-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:da1885faca29091f9e408c0cc6b43a0b29a2128acf8d08c188febc5d9f99129d"}, + {file = "lupa-1.13-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4525e954e951562eb5609eca6ac694d0158a5351649656e50d524f87f71e2a35"}, + {file = "lupa-1.13-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5a04febcd3016cb992e6c5b2f97834ad53a2fd4b37767d9afdce116021c2463a"}, + {file = "lupa-1.13-cp27-cp27m-win32.whl", hash = "sha256:98f6d3debc4d3668e5e19d70e288dbdbbedef021a75ac2e42c450c7679b4bf52"}, + {file = "lupa-1.13-cp27-cp27m-win_amd64.whl", hash = "sha256:7009719bf65549c018a2f925ff06b9d862a5a1e22f8a7aeeef807eb1e99b56bc"}, + {file = "lupa-1.13-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bde9e73b06d147d31b970123a013cc6d28a4bea7b3d6b64fe115650cbc62b1a3"}, + {file = "lupa-1.13-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a122baad6c6f9aaae496a59318217c068ae73654f618526e404a28775b46da38"}, + {file = "lupa-1.13-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:4d1588486ed16d6b53f41b080047d44db3aa9991cf8a30da844cb97486a63c8b"}, + {file = "lupa-1.13-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:a79be3ca652c8392d612bdc2234074325a68ec572c4175a35347cd650ef4a4b9"}, + {file = "lupa-1.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d9105f3b098cd4c276d6258f8254224243066f51c5d3c923b8f460efac9de37b"}, + {file = "lupa-1.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:2d1fbddfa2914c405004f805afb13f5fc385793f3ba28e86a6f0c85b4059b86c"}, + {file = "lupa-1.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a3c84994399887a8befc82aef4d837582db45a301413025c510e20fef9e9148"}, + {file = "lupa-1.13-cp310-cp310-win32.whl", hash = "sha256:c665af2a92e79106045f973174e0849f92b44395f5247505d321bc1173d9f3fd"}, + {file = "lupa-1.13-cp310-cp310-win_amd64.whl", hash = "sha256:c9b47a9e93cb8e8f342343f4e0963eb1966d36baeced482575141925eafc17dc"}, + {file = "lupa-1.13-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:b3003d723faabb9502259662722462cbff368f26ed83a6311f65949d298593bf"}, + {file = "lupa-1.13-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b341b8a4711558af771bd4a954a6ffe531bfe097c1f1cdce84b9ad56070dfe90"}, + {file = "lupa-1.13-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ea049ee507a549eec553a9d27e3e6c034eae8c145e7bad5947e85c4b9e23757b"}, + {file = "lupa-1.13-cp35-cp35m-win32.whl", hash = "sha256:ba6c49646ad42c836f18ff8f1b6b8db4ca32fc02e786e1bf401b0fa34fe82cca"}, + {file = "lupa-1.13-cp35-cp35m-win_amd64.whl", hash = "sha256:de51177d1374fd9cce27b9cdb20771142d91a509e42337b3e7c6cffbba818d6f"}, + {file = "lupa-1.13-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:dddfeb031ab67c8bdbeefd2de237a98bee58e2166d5ed629c3a0c3842bb91738"}, + {file = "lupa-1.13-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57f00004c185bd60459586a9d08961541f5da1cfec5925a3fc1ab68deaa2e038"}, + {file = "lupa-1.13-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a940be5b38b68b344691558ffde1b44377ad66c105661f6f58c7d4c0c227d8ea"}, + {file = "lupa-1.13-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:807b27c13f7598af9343455204a6a23b6b919180f01668c9b8fa4f9b0d75dedb"}, + {file = "lupa-1.13-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a52d5a8305f4854f91ee39f5ee6f175f4d38f362c6b00483fe618ae6f9dff5b"}, + {file = "lupa-1.13-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0ad47549359df03b3e59796ba09df548e1fd046f9245391dae79699c9ffec0f6"}, + {file = "lupa-1.13-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fbf99cea003b38a146dff5333ba58edb8165e01c42f15d7f76fdb72e761b5827"}, + {file = "lupa-1.13-cp36-cp36m-win32.whl", hash = "sha256:a101c84097fdfa7b1a38f9d5a3055759da4e222c255ab8e5ac5b683704e62c97"}, + {file = "lupa-1.13-cp36-cp36m-win_amd64.whl", hash = "sha256:00376b3bcb00bb57e067740ea9ff00f610a44aff5338ea93d3198a035f8965c6"}, + {file = "lupa-1.13-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:91001c9667d60b69c3ad623dc315d7b59712e1617fe6204e5852c31cda778678"}, + {file = "lupa-1.13-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:65c9d034d7215e8929a4ab48c9d9d372786ef47c8e61c294851bf0b8f5b4fbf4"}, + {file = "lupa-1.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:928527222b2a15bd3dcea646f7585852097302c078c338fb0f184ce560d48c6c"}, + {file = "lupa-1.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:5e157d97e379931a7fa90d9afa66600f796960bc062e04a9bb37f24fa7c5c967"}, + {file = "lupa-1.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a67336d542d71e095c07dacc72c16158745ae4ef08e8a7bfe75827da604b4979"}, + {file = "lupa-1.13-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0c5cd027c998db5b29ca8dd956c255d50914aed614d1c9edb68bc3315f916f59"}, + {file = "lupa-1.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:76b06355f0b3d3aece5c38d20a66ab7d3046add95b8d04b677ade162fce2ffd0"}, + {file = "lupa-1.13-cp37-cp37m-win32.whl", hash = "sha256:2a6b0a7e45390de36d11dd8705b2a0a10739ba8ed2e99c130e983ad72d56ddc9"}, + {file = "lupa-1.13-cp37-cp37m-win_amd64.whl", hash = "sha256:42ffbe43119225cc58c7ebd2210123b9367b098ac25a7f0ef5d473e2f65fc0d9"}, + {file = "lupa-1.13-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:7ff445a5d8ab25e623f871c600af58f1cd6207f6873a42c3b8c1683f13a22db0"}, + {file = "lupa-1.13-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:dd0404f11b9473372fe2a8bdf0d64b361852ae08699d6dcde1215db3bd6c7b9c"}, + {file = "lupa-1.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:14419b29152667fb2d78c6d5176f9a704c765aeecb80fe6c079a8dba9f864529"}, + {file = "lupa-1.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:9e644032b40b59420ffa0d58ca1705351785ce8e39b77d9f1a8c4cf78e371adb"}, + {file = "lupa-1.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c090991e2b701ded6c9e330ea582a74dd9cb09069b3de9ae897b938bd97dc98f"}, + {file = "lupa-1.13-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6812f16530a1dc88f66c76a002e1c16039d3d98e1ff283a2efd5a492342ba00c"}, + {file = "lupa-1.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ff3989ab562fb62e9df2290739c7f82e05d5ba7d2fa2ea319991885dfc818c81"}, + {file = "lupa-1.13-cp38-cp38-win32.whl", hash = "sha256:48fa15cf24d297c50f21bff1fe1883f7a6a15b34b70db5a6c18d2dfbed6b6e16"}, + {file = "lupa-1.13-cp38-cp38-win_amd64.whl", hash = "sha256:ea32a62d404c3d9e119e83b653aa56c034cae63a4e830aefa15bf3a25299b29e"}, + {file = "lupa-1.13-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:80d36fbdc6218332232b4c214a2f9c36b13136b546dca0b3d19aca12d77e1f8e"}, + {file = "lupa-1.13-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:db4745132f8abe0c9daac155af9d196926c9e10662d999edd805756d91502a01"}, + {file = "lupa-1.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:938fb12c556737f9e4ffb7912540e35423d1be3166c6d4099ca4f3e177fe619e"}, + {file = "lupa-1.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:de913a471ee6dc86435b647dda3cdb787990b164d8c8c63ca03d6e934f305a55"}, + {file = "lupa-1.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:488d1bd773f10331ca67b0914c880900316634fd14538f76c3c2fbc7e6b56043"}, + {file = "lupa-1.13-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dc101e6d82ffa1b3fcfc77f2430a10c02def972cf0f8c7a229e272697e22e35c"}, + {file = "lupa-1.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:361a55883b692d25478a69104d8ecce4cad058ba39ec1b7378b1209f86867687"}, + {file = "lupa-1.13-cp39-cp39-win32.whl", hash = "sha256:9a6cd192e789fbc7f6a777a17b5b517c447a6dc6049e60c1becb300f86205345"}, + {file = "lupa-1.13-cp39-cp39-win_amd64.whl", hash = "sha256:9fe47cda7cc81bd9b111f1317ed60e3da2620f4fef5360b690dcf62f88bbc668"}, + {file = "lupa-1.13-pp37-pypy37_pp73-macosx_10_14_x86_64.whl", hash = "sha256:7d860dc0062b3001993355b12b939f68e0e2871a19a81427d2a9ced893574b58"}, + {file = "lupa-1.13-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6c0358386f16afb50145b143774791c942c93a9721078a17983486a2d9f8f45b"}, + {file = "lupa-1.13-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:a46962ebdc6278e82520c66d5dd1eed50099aa2f56b6827b7a4f001664d9ad1d"}, + {file = "lupa-1.13-pp37-pypy37_pp73-win32.whl", hash = "sha256:436daf32385bcb9b6b9f922cbc0b64d133db141f0f7d8946a3a653e83b478713"}, + {file = "lupa-1.13-pp38-pypy38_pp73-macosx_10_14_x86_64.whl", hash = "sha256:f1165e89aa8d2a0644619517e04410b9f5e3da2c9b3d105bf53f70e786f91f79"}, + {file = "lupa-1.13-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:325069e4f3cf4b1232d03fb330ba1449867fc7dd727ecebaf0e602ddcacaf9d4"}, + {file = "lupa-1.13-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:ce59c335b80ec4f9e98181970c18552f51adba5c3380ef5d46bdb3246b87963d"}, + {file = "lupa-1.13-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:ad263ba6e54a13ac036364ae43ba7613c869c5ee6ff7dbb86791685a6cba13c5"}, + {file = "lupa-1.13-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:86f4f46ee854e36cf5b6cf2317075023f395eede53efec0a694bc4a01fc03ab7"}, + {file = "lupa-1.13-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:59799f40774dd5b8cfb99b11d6ce3a3f3a141e112472874389d47c81a7377ef9"}, + {file = "lupa-1.13.tar.gz", hash = "sha256:e1d94ac2a630d271027dac2c21d1428771d9ea9d4d88f15f20a7781340f02a4e"}, +] lxml = [] -markdownify = [] -mccabe = [] -more-itertools = [] -mslex = [] -multidict = [] +markdownify = [ + {file = "markdownify-0.6.1-py3-none-any.whl", hash = "sha256:7489fd5c601536996a376c4afbcd1dd034db7690af807120681461e82fbc0acc"}, + {file = "markdownify-0.6.1.tar.gz", hash = "sha256:31d7c13ac2ada8bfc7535a25fee6622ca720e1b5f2d4a9cbc429d167c21f886d"}, +] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] +more-itertools = [ + {file = "more-itertools-8.12.0.tar.gz", hash = "sha256:7dc6ad46f05f545f900dd59e8dfb4e84a4827b97b3cfecb175ea0c7d247f6064"}, + {file = "more_itertools-8.12.0-py3-none-any.whl", hash = "sha256:43e6dd9942dffd72661a2c4ef383ad7da1e6a3e968a927ad7a6083ab410a688b"}, +] +mslex = [ + {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, + {file = "mslex-0.3.0.tar.gz", hash = "sha256:4a1ac3f25025cad78ad2fe499dd16d42759f7a3801645399cce5c404415daa97"}, +] +multidict = [ + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdda29a3c7e76a064f2477c9aab1ba96fd94e02e386f1e665bca1807fc5386f"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3368bf2398b0e0fcbf46d85795adc4c259299fec50c1416d0f77c0a843a3eed9"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4f052ee022928d34fe1f4d2bc743f32609fb79ed9c49a1710a5ad6b2198db20"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:225383a6603c086e6cef0f2f05564acb4f4d5f019a4e3e983f572b8530f70c88"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50bd442726e288e884f7be9071016c15a8742eb689a593a0cac49ea093eef0a7"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:47e6a7e923e9cada7c139531feac59448f1f47727a79076c0b1ee80274cd8eee"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0556a1d4ea2d949efe5fd76a09b4a82e3a4a30700553a6725535098d8d9fb672"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:626fe10ac87851f4cffecee161fc6f8f9853f0f6f1035b59337a51d29ff3b4f9"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8064b7c6f0af936a741ea1efd18690bacfbae4078c0c385d7c3f611d11f0cf87"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2d36e929d7f6a16d4eb11b250719c39560dd70545356365b494249e2186bc389"}, + {file = "multidict-6.0.2-cp310-cp310-win32.whl", hash = "sha256:fcb91630817aa8b9bc4a74023e4198480587269c272c58b3279875ed7235c293"}, + {file = "multidict-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:8cbf0132f3de7cc6c6ce00147cc78e6439ea736cee6bca4f068bcf892b0fd658"}, + {file = "multidict-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:05f6949d6169878a03e607a21e3b862eaf8e356590e8bdae4227eedadacf6e51"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2c2e459f7050aeb7c1b1276763364884595d47000c1cddb51764c0d8976e608"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0509e469d48940147e1235d994cd849a8f8195e0bca65f8f5439c56e17872a3"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:514fe2b8d750d6cdb4712346a2c5084a80220821a3e91f3f71eec11cf8d28fd4"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19adcfc2a7197cdc3987044e3f415168fc5dc1f720c932eb1ef4f71a2067e08b"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9d153e7f1f9ba0b23ad1568b3b9e17301e23b042c23870f9ee0522dc5cc79e8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aef9cc3d9c7d63d924adac329c33835e0243b5052a6dfcbf7732a921c6e918ba"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4571f1beddff25f3e925eea34268422622963cd8dc395bb8778eb28418248e43"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:d48b8ee1d4068561ce8033d2c344cf5232cb29ee1a0206a7b828c79cbc5982b8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:45183c96ddf61bf96d2684d9fbaf6f3564d86b34cb125761f9a0ef9e36c1d55b"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:75bdf08716edde767b09e76829db8c1e5ca9d8bb0a8d4bd94ae1eafe3dac5e15"}, + {file = "multidict-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:a45e1135cb07086833ce969555df39149680e5471c04dfd6a915abd2fc3f6dbc"}, + {file = "multidict-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6f3cdef8a247d1eafa649085812f8a310e728bdf3900ff6c434eafb2d443b23a"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e875b6086e325bab7e680e4316d667fc0e5e174bb5611eb16b3ea121c8951b86"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feea820722e69451743a3d56ad74948b68bf456984d63c1a92e8347b7b88452d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc57c68cb9139c7cd6fc39f211b02198e69fb90ce4bc4a094cf5fe0d20fd8b0"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:497988d6b6ec6ed6f87030ec03280b696ca47dbf0648045e4e1d28b80346560d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89171b2c769e03a953d5969b2f272efa931426355b6c0cb508022976a17fd376"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684133b1e1fe91eda8fa7447f137c9490a064c6b7f392aa857bba83a28cfb693"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9fc9c4849a07f3635ccffa895d57abce554b467d611a5009ba4f39b78a8849"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e07c8e79d6e6fd37b42f3250dba122053fddb319e84b55dd3a8d6446e1a7ee49"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4070613ea2227da2bfb2c35a6041e4371b0af6b0be57f424fe2318b42a748516"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:47fbeedbf94bed6547d3aa632075d804867a352d86688c04e606971595460227"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5774d9218d77befa7b70d836004a768fb9aa4fdb53c97498f4d8d3f67bb9cfa9"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2957489cba47c2539a8eb7ab32ff49101439ccf78eab724c828c1a54ff3ff98d"}, + {file = "multidict-6.0.2-cp38-cp38-win32.whl", hash = "sha256:e5b20e9599ba74391ca0cfbd7b328fcc20976823ba19bc573983a25b32e92b57"}, + {file = "multidict-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8004dca28e15b86d1b1372515f32eb6f814bdf6f00952699bdeb541691091f96"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2e4a0785b84fb59e43c18a015ffc575ba93f7d1dbd272b4cdad9f5134b8a006c"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6701bf8a5d03a43375909ac91b6980aea74b0f5402fbe9428fc3f6edf5d9677e"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a007b1638e148c3cfb6bf0bdc4f82776cef0ac487191d093cdc316905e504071"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07a017cfa00c9890011628eab2503bee5872f27144936a52eaab449be5eaf032"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c207fff63adcdf5a485969131dc70e4b194327666b7e8a87a97fbc4fd80a53b2"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:373ba9d1d061c76462d74e7de1c0c8e267e9791ee8cfefcf6b0b2495762c370c"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfba7c6d5d7c9099ba21f84662b037a0ffd4a5e6b26ac07d19e423e6fdf965a9"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19d9bad105dfb34eb539c97b132057a4e709919ec4dd883ece5838bcbf262b80"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:de989b195c3d636ba000ee4281cd03bb1234635b124bf4cd89eeee9ca8fcb09d"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7c40b7bbece294ae3a87c1bc2abff0ff9beef41d14188cda94ada7bcea99b0fb"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:d16cce709ebfadc91278a1c005e3c17dd5f71f5098bfae1035149785ea6e9c68"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a2c34a93e1d2aa35fbf1485e5010337c72c6791407d03aa5f4eed920343dd360"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:feba80698173761cddd814fa22e88b0661e98cb810f9f986c54aa34d281e4937"}, + {file = "multidict-6.0.2-cp39-cp39-win32.whl", hash = "sha256:23b616fdc3c74c9fe01d76ce0d1ce872d2d396d8fa8e4899398ad64fb5aa214a"}, + {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, + {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, +] nodeenv = [] -ordered-set = [] -packaging = [] -pep8-naming = [] -pip-licenses = [] -platformdirs = [] -pluggy = [] -pre-commit = [] +ordered-set = [ + {file = "ordered-set-4.0.2.tar.gz", hash = "sha256:ba93b2df055bca202116ec44b9bead3df33ea63a7d5827ff8e16738b97f33a95"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +pep8-naming = [ + {file = "pep8-naming-0.12.1.tar.gz", hash = "sha256:bb2455947757d162aa4cad55dba4ce029005cd1692f2899a21d51d8630ca7841"}, + {file = "pep8_naming-0.12.1-py2.py3-none-any.whl", hash = "sha256:4a8daeaeb33cfcde779309fc0c9c0a68a3bbe2ad8a8308b763c5068f86eb9f37"}, +] +pip-licenses = [ + {file = "pip-licenses-3.5.3.tar.gz", hash = "sha256:f44860e00957b791c6c6005a3328f2d5eaeee96ddb8e7d87d4b0aa25b02252e4"}, + {file = "pip_licenses-3.5.3-py3-none-any.whl", hash = "sha256:59c148d6a03784bf945d232c0dc0e9de4272a3675acaa0361ad7712398ca86ba"}, +] +platformdirs = [ + {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, + {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +pre-commit = [ + {file = "pre_commit-2.17.0-py2.py3-none-any.whl", hash = "sha256:725fa7459782d7bec5ead072810e47351de01709be838c2ce1726b9591dad616"}, + {file = "pre_commit-2.17.0.tar.gz", hash = "sha256:c1a8040ff15ad3d648c70cc3e55b93e4d2d5b687320955505587fd79bbaed06a"}, +] psutil = [] -ptable = [] -py = [] +ptable = [ + {file = "PTable-0.9.2.tar.gz", hash = "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292"}, +] +py = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] pycares = [] -pycodestyle = [] -pycparser = [] -pydocstyle = [] -pyflakes = [] +pycodestyle = [ + {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, + {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, +] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] +pydantic = [] +pydocstyle = [ + {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, + {file = "pydocstyle-6.1.1.tar.gz", hash = "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc"}, +] +pyflakes = [ + {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, + {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, +] pyparsing = [] -pyreadline3 = [] -pytest = [] -pytest-cov = [] -pytest-forked = [] -pytest-xdist = [] -python-dateutil = [] -python-dotenv = [] -python-frontmatter = [] -pyyaml = [] -rapidfuzz = [] +pyreadline3 = [ + {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, + {file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"}, +] +pytest = [ + {file = "pytest-7.1.1-py3-none-any.whl", hash = "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea"}, + {file = "pytest-7.1.1.tar.gz", hash = "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63"}, +] +pytest-cov = [ + {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, + {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, +] +pytest-forked = [ + {file = "pytest-forked-1.4.0.tar.gz", hash = "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e"}, + {file = "pytest_forked-1.4.0-py3-none-any.whl", hash = "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8"}, +] +pytest-xdist = [ + {file = "pytest-xdist-2.5.0.tar.gz", hash = "sha256:4580deca3ff04ddb2ac53eba39d76cb5dd5edeac050cb6fbc768b0dd712b4edf"}, + {file = "pytest_xdist-2.5.0-py3-none-any.whl", hash = "sha256:6fe5c74fec98906deb8f2d2b616b5c782022744978e7bd4695d39c8f42d0ce65"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +python-dotenv = [ + {file = "python-dotenv-0.20.0.tar.gz", hash = "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f"}, + {file = "python_dotenv-0.20.0-py3-none-any.whl", hash = "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"}, +] +python-frontmatter = [ + {file = "python-frontmatter-1.0.0.tar.gz", hash = "sha256:e98152e977225ddafea6f01f40b4b0f1de175766322004c826ca99842d19a7cd"}, + {file = "python_frontmatter-1.0.0-py3-none-any.whl", hash = "sha256:766ae75f1b301ffc5fe3494339147e0fd80bc3deff3d7590a93991978b579b08"}, +] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +rapidfuzz = [ + {file = "rapidfuzz-2.0.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b306b4a1d42a8dfd5f3daff9a82853f1541e5c74a2ec34515a5e5cd51f3c7307"}, + {file = "rapidfuzz-2.0.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2ee380254d8b29d0b0f47a020e7f16375a4d97164b8071b3f94d5c684d744093"}, + {file = "rapidfuzz-2.0.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac993b8760c5308d885c300355e2c537daf0696ebc5d30436af83818978e661c"}, + {file = "rapidfuzz-2.0.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d06a394e475316aeddbf4bf9691aabf4825f8c1acf87b49abbb7b9dad7e555ae"}, + {file = "rapidfuzz-2.0.7-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79883fcfc3e550b356d45ac2bf1af391161f9ddb64b1ed504f9a94086b824709"}, + {file = "rapidfuzz-2.0.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d44d74ace68b3ec6dee4501188c74f124da8c940877989baf9f672d51368e171"}, + {file = "rapidfuzz-2.0.7-cp310-cp310-win32.whl", hash = "sha256:9ec9fd78d40f392cd4ce91dbb17477cd07740d0cb0b7bf44e9ab67c16ee3d5ce"}, + {file = "rapidfuzz-2.0.7-cp310-cp310-win_amd64.whl", hash = "sha256:7983ed01b0ac5343bea4d737024576a86a8c68f3c8d811498eb0facf8d3bafc1"}, + {file = "rapidfuzz-2.0.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:49fd3d2a789abc30c811d6ed81db1c5f143caf5e975720bf9ab62c920253d5e9"}, + {file = "rapidfuzz-2.0.7-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:636489517bbd0786f300948f8eba59635f2fb781ecbc2ed19deba3426ee32ab6"}, + {file = "rapidfuzz-2.0.7-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c6a47418b86a6b8267a89f253e2b14f9aa8b4b559141b15f8c8a9769d19b109"}, + {file = "rapidfuzz-2.0.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fe01ca2cbdb2aee6f80c1fc3a82fa69ee9ef9c44f085a725113b5d12209e05d"}, + {file = "rapidfuzz-2.0.7-cp36-cp36m-win32.whl", hash = "sha256:8157406a1b44cd742d65c65ca8345e47fcc8642148a970626b886fb52b3abd1d"}, + {file = "rapidfuzz-2.0.7-cp36-cp36m-win_amd64.whl", hash = "sha256:3062ea2a0481196376e364470c682d5ebc22eb5d4c114350f05f079119ea61b8"}, + {file = "rapidfuzz-2.0.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cbfc3fcbbd00edf7f917ad0d6bf46350c64a9910c14d05e1936d436170f2531d"}, + {file = "rapidfuzz-2.0.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae15eb44e014101b208c97a253d850d6fb4a8465f3c9ee8be3508b03135ad0e7"}, + {file = "rapidfuzz-2.0.7-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9224115aae07d42b9250d8ca58d5568cab2ddd8720c551aa7de9dcec661ee86"}, + {file = "rapidfuzz-2.0.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42bc2cf64ebbf2a80e6fd03353679de17118a431dce358cfadc7cdb72ac9510a"}, + {file = "rapidfuzz-2.0.7-cp37-cp37m-win32.whl", hash = "sha256:34416ee6265dfa1415e9f10c7dafe6a85296117f534f67d00021eeaa661c8d9e"}, + {file = "rapidfuzz-2.0.7-cp37-cp37m-win_amd64.whl", hash = "sha256:4044ef50f020f16f99b5979784b648b7ab90cd6bd0d275359818a2c155f9c01d"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1332fb51345e431ba39e075c3dbc222bb9770f0e73c097c7a65c8c2ea331004c"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2ac560a603d0d1b9d70cc0a376d1adf57ece4195e61351d410e0c7b0fa280cbe"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0fd757a38e14f247d929af7df6762aee2082f7a6882c85a31f17b09a450bbb5e"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723a48d5937e4a558fb5df553b3d0e0b3cc05de7f7a8d43a920682b796010ab5"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c9b344e3f69c5b69ae0c96411d3ee1dab02ec49124471e44ce2a16f6446fa6d"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f34f905a0e9fa01cf26b9208daac6523708f9439958397b21b95c6c4fe508b"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-win32.whl", hash = "sha256:a95a45939cbd035c2d4779765a81485215a12fa5f1b912c2738374fad93e753d"}, + {file = "rapidfuzz-2.0.7-cp38-cp38-win_amd64.whl", hash = "sha256:14234ecc57e1799e24c9dcd230bba02630c4f38ca60c0eb075452313da8e0e95"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a9959374974fb96d3941334f5f8caeea971ea9718279514748c53d381146c5a7"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2a988b5ff46823e0d5e14b4a1cce3ef13024009115df61d1d3b7ba14678f421"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:603d179205972ebb5b01e7a84ead465d08813d50401216d5cc81fc2589e2c957"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a934734aa247f57c683932ae0d38653063b2d97540598b551294b40ff242bd62"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c42064174035f3633f4a815c38a76514875ca8531fac3f992202a41d1f338a41"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46a46b8bab2ceee4877dfb281e94a43197b118d96cb04325e07540f7f9c57324"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-win32.whl", hash = "sha256:1f892f3dd0acfbc2ba0b90d72cac42dd468ac9a8f7ac2179c91c29c22a4f7960"}, + {file = "rapidfuzz-2.0.7-cp39-cp39-win_amd64.whl", hash = "sha256:233024373cb77dc2ef510b5fccac0429edb3294ea631ad777a7e3ff614501578"}, + {file = "rapidfuzz-2.0.7-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be8121175e7096062a312b73823385389635c4dec50a9e0496b29c4ba0b50362"}, + {file = "rapidfuzz-2.0.7-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ad5282cf9921c6dbfe1c58e5af05c3014eabc20afd8fafcc0e6a56e9263875a0"}, + {file = "rapidfuzz-2.0.7-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c67d650e25a7c281127865cc50c3588d5319200c8a11837df51ab3eead7cf066"}, + {file = "rapidfuzz-2.0.7-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07ccd298a24de2dadead47e75f23ff747ed3ee551964a8401ccae31a577cebb1"}, + {file = "rapidfuzz-2.0.7-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1e70ec13c00a9f28cce76a29eb5c4e6aeb5dadb9ddb35b74dfe05d503c09a4a"}, + {file = "rapidfuzz-2.0.7-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c47fda63c0d9d8275b319cdc226f96b3f1c16a395409442bff566b6de6b7cac9"}, + {file = "rapidfuzz-2.0.7.tar.gz", hash = "sha256:93bf42784fd74ebf1a8e89ca1596e9bea7f3ac4a61b825ecc6eb2d9893ad6844"}, +] redis = [] -regex = [] +regex = [ + {file = "regex-2022.3.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:42eb13b93765c6698a5ab3bcd318d8c39bb42e5fa8a7fcf7d8d98923f3babdb1"}, + {file = "regex-2022.3.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9beb03ff6fe509d6455971c2489dceb31687b38781206bcec8e68bdfcf5f1db2"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0a5a1fdc9f148a8827d55b05425801acebeeefc9e86065c7ac8b8cc740a91ff"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cb374a2a4dba7c4be0b19dc7b1adc50e6c2c26c3369ac629f50f3c198f3743a4"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c33ce0c665dd325200209340a88438ba7a470bd5f09f7424e520e1a3ff835b52"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04c09b9651fa814eeeb38e029dc1ae83149203e4eeb94e52bb868fadf64852bc"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab5d89cfaf71807da93c131bb7a19c3e19eaefd613d14f3bce4e97de830b15df"}, + {file = "regex-2022.3.15-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e2630ae470d6a9f8e4967388c1eda4762706f5750ecf387785e0df63a4cc5af"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:df037c01d68d1958dad3463e2881d3638a0d6693483f58ad41001aa53a83fcea"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:940570c1a305bac10e8b2bc934b85a7709c649317dd16520471e85660275083a"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7f63877c87552992894ea1444378b9c3a1d80819880ae226bb30b04789c0828c"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3e265b388cc80c7c9c01bb4f26c9e536c40b2c05b7231fbb347381a2e1c8bf43"}, + {file = "regex-2022.3.15-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:058054c7a54428d5c3e3739ac1e363dc9347d15e64833817797dc4f01fb94bb8"}, + {file = "regex-2022.3.15-cp310-cp310-win32.whl", hash = "sha256:76435a92e444e5b8f346aed76801db1c1e5176c4c7e17daba074fbb46cb8d783"}, + {file = "regex-2022.3.15-cp310-cp310-win_amd64.whl", hash = "sha256:174d964bc683b1e8b0970e1325f75e6242786a92a22cedb2a6ec3e4ae25358bd"}, + {file = "regex-2022.3.15-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6e1d8ed9e61f37881c8db383a124829a6e8114a69bd3377a25aecaeb9b3538f8"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b52771f05cff7517f7067fef19ffe545b1f05959e440d42247a17cd9bddae11b"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:673f5a393d603c34477dbad70db30025ccd23996a2d0916e942aac91cc42b31a"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8923e1c5231549fee78ff9b2914fad25f2e3517572bb34bfaa3aea682a758683"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:764e66a0e382829f6ad3bbce0987153080a511c19eb3d2f8ead3f766d14433ac"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd00859291658fe1fda48a99559fb34da891c50385b0bfb35b808f98956ef1e7"}, + {file = "regex-2022.3.15-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa2ce79f3889720b46e0aaba338148a1069aea55fda2c29e0626b4db20d9fcb7"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:34bb30c095342797608727baf5c8aa122406aa5edfa12107b8e08eb432d4c5d7"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:25ecb1dffc5e409ca42f01a2b2437f93024ff1612c1e7983bad9ee191a5e8828"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:aa5eedfc2461c16a092a2fabc5895f159915f25731740c9152a1b00f4bcf629a"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:7d1a6e403ac8f1d91d8f51c441c3f99367488ed822bda2b40836690d5d0059f5"}, + {file = "regex-2022.3.15-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:3e4d710ff6539026e49f15a3797c6b1053573c2b65210373ef0eec24480b900b"}, + {file = "regex-2022.3.15-cp36-cp36m-win32.whl", hash = "sha256:0100f0ded953b6b17f18207907159ba9be3159649ad2d9b15535a74de70359d3"}, + {file = "regex-2022.3.15-cp36-cp36m-win_amd64.whl", hash = "sha256:f320c070dea3f20c11213e56dbbd7294c05743417cde01392148964b7bc2d31a"}, + {file = "regex-2022.3.15-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fc8c7958d14e8270171b3d72792b609c057ec0fa17d507729835b5cff6b7f69a"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ca6dcd17f537e9f3793cdde20ac6076af51b2bd8ad5fe69fa54373b17b48d3c"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0214ff6dff1b5a4b4740cfe6e47f2c4c92ba2938fca7abbea1359036305c132f"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a98ae493e4e80b3ded6503ff087a8492db058e9c68de371ac3df78e88360b374"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b1cc70e31aacc152a12b39245974c8fccf313187eead559ee5966d50e1b5817"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4829db3737480a9d5bfb1c0320c4ee13736f555f53a056aacc874f140e98f64"}, + {file = "regex-2022.3.15-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:303b15a3d32bf5fe5a73288c316bac5807587f193ceee4eb6d96ee38663789fa"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:dc7b7c16a519d924c50876fb152af661a20749dcbf653c8759e715c1a7a95b18"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ce3057777a14a9a1399b81eca6a6bfc9612047811234398b84c54aeff6d536ea"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:48081b6bff550fe10bcc20c01cf6c83dbca2ccf74eeacbfac240264775fd7ecf"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dcbb7665a9db9f8d7642171152c45da60e16c4f706191d66a1dc47ec9f820aed"}, + {file = "regex-2022.3.15-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c155a1a80c5e7a8fa1d9bb1bf3c8a953532b53ab1196092749bafb9d3a7cbb60"}, + {file = "regex-2022.3.15-cp37-cp37m-win32.whl", hash = "sha256:04b5ee2b6d29b4a99d38a6469aa1db65bb79d283186e8460542c517da195a8f6"}, + {file = "regex-2022.3.15-cp37-cp37m-win_amd64.whl", hash = "sha256:797437e6024dc1589163675ae82f303103063a0a580c6fd8d0b9a0a6708da29e"}, + {file = "regex-2022.3.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8afcd1c2297bc989dceaa0379ba15a6df16da69493635e53431d2d0c30356086"}, + {file = "regex-2022.3.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0066a6631c92774391f2ea0f90268f0d82fffe39cb946f0f9c6b382a1c61a5e5"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8248f19a878c72d8c0a785a2cd45d69432e443c9f10ab924c29adda77b324ae"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8d1f3ea0d1924feb4cf6afb2699259f658a08ac6f8f3a4a806661c2dfcd66db1"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:794a6bc66c43db8ed06698fc32aaeaac5c4812d9f825e9589e56f311da7becd9"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d1445824944e642ffa54c4f512da17a953699c563a356d8b8cbdad26d3b7598"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f553a1190ae6cd26e553a79f6b6cfba7b8f304da2071052fa33469da075ea625"}, + {file = "regex-2022.3.15-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:75a5e6ce18982f0713c4bac0704bf3f65eed9b277edd3fb9d2b0ff1815943327"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f16cf7e4e1bf88fecf7f41da4061f181a6170e179d956420f84e700fb8a3fd6b"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dad3991f0678facca1a0831ec1ddece2eb4d1dd0f5150acb9440f73a3b863907"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:491fc754428514750ab21c2d294486223ce7385446f2c2f5df87ddbed32979ae"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:6504c22c173bb74075d7479852356bb7ca80e28c8e548d4d630a104f231e04fb"}, + {file = "regex-2022.3.15-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:01c913cf573d1da0b34c9001a94977273b5ee2fe4cb222a5d5b320f3a9d1a835"}, + {file = "regex-2022.3.15-cp38-cp38-win32.whl", hash = "sha256:029e9e7e0d4d7c3446aa92474cbb07dafb0b2ef1d5ca8365f059998c010600e6"}, + {file = "regex-2022.3.15-cp38-cp38-win_amd64.whl", hash = "sha256:947a8525c0a95ba8dc873191f9017d1b1e3024d4dc757f694e0af3026e34044a"}, + {file = "regex-2022.3.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:591d4fba554f24bfa0421ba040cd199210a24301f923ed4b628e1e15a1001ff4"}, + {file = "regex-2022.3.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b9809404528a999cf02a400ee5677c81959bc5cb938fdc696b62eb40214e3632"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f08a7e4d62ea2a45557f561eea87c907222575ca2134180b6974f8ac81e24f06"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a86cac984da35377ca9ac5e2e0589bd11b3aebb61801204bd99c41fac516f0d"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:286908cbe86b1a0240a867aecfe26a439b16a1f585d2de133540549831f8e774"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b7494df3fdcc95a1f76cf134d00b54962dd83189520fd35b8fcd474c0aa616d"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b1ceede92400b3acfebc1425937454aaf2c62cd5261a3fabd560c61e74f6da3"}, + {file = "regex-2022.3.15-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0317eb6331146c524751354ebef76a7a531853d7207a4d760dfb5f553137a2a4"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9c144405220c5ad3f5deab4c77f3e80d52e83804a6b48b6bed3d81a9a0238e4c"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5b2e24f3ae03af3d8e8e6d824c891fea0ca9035c5d06ac194a2700373861a15c"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f2c53f3af011393ab5ed9ab640fa0876757498aac188f782a0c620e33faa2a3d"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:060f9066d2177905203516c62c8ea0066c16c7342971d54204d4e51b13dfbe2e"}, + {file = "regex-2022.3.15-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:530a3a16e57bd3ea0dff5ec2695c09632c9d6c549f5869d6cf639f5f7153fb9c"}, + {file = "regex-2022.3.15-cp39-cp39-win32.whl", hash = "sha256:78ce90c50d0ec970bd0002462430e00d1ecfd1255218d52d08b3a143fe4bde18"}, + {file = "regex-2022.3.15-cp39-cp39-win_amd64.whl", hash = "sha256:c5adc854764732dbd95a713f2e6c3e914e17f2ccdc331b9ecb777484c31f73b6"}, + {file = "regex-2022.3.15.tar.gz", hash = "sha256:0a7b75cc7bb4cc0334380053e4671c560e31272c9d2d5a6c4b8e9ae2c9bd0f82"}, +] requests = [] -requests-file = [] -sentry-sdk = [] -sgmllib3k = [] -six = [] -snowballstemmer = [] -sortedcontainers = [] -soupsieve = [] -statsd = [] -taskipy = [] -testfixtures = [] -tldextract = [] -toml = [] -tomli = [] +requests-file = [ + {file = "requests-file-1.5.1.tar.gz", hash = "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e"}, + {file = "requests_file-1.5.1-py2.py3-none-any.whl", hash = "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953"}, +] +sentry-sdk = [ + {file = "sentry-sdk-1.5.8.tar.gz", hash = "sha256:38fd16a92b5ef94203db3ece10e03bdaa291481dd7e00e77a148aa0302267d47"}, + {file = "sentry_sdk-1.5.8-py2.py3-none-any.whl", hash = "sha256:32af1a57954576709242beb8c373b3dbde346ac6bd616921def29d68846fb8c3"}, +] +sgmllib3k = [ + {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +snowballstemmer = [ + {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, + {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, +] +sortedcontainers = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] +soupsieve = [ + {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, + {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, +] +statsd = [ + {file = "statsd-3.3.0-py2.py3-none-any.whl", hash = "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa"}, + {file = "statsd-3.3.0.tar.gz", hash = "sha256:e3e6db4c246f7c59003e51c9720a51a7f39a396541cb9b147ff4b14d15b5dd1f"}, +] +taskipy = [ + {file = "taskipy-1.10.1-py3-none-any.whl", hash = "sha256:9b38333654da487b6d16de6fa330b7629d1935d1e74819ba4c5f17a1c372d37b"}, + {file = "taskipy-1.10.1.tar.gz", hash = "sha256:6fa0b11c43d103e376063e90be31d87b435aad50fb7dc1c9a2de9b60a85015ed"}, +] +testfixtures = [ + {file = "testfixtures-6.18.5-py2.py3-none-any.whl", hash = "sha256:7de200e24f50a4a5d6da7019fb1197aaf5abd475efb2ec2422fdcf2f2eb98c1d"}, + {file = "testfixtures-6.18.5.tar.gz", hash = "sha256:02dae883f567f5b70fd3ad3c9eefb95912e78ac90be6c7444b5e2f46bf572c84"}, +] +tldextract = [ + {file = "tldextract-3.2.0-py3-none-any.whl", hash = "sha256:427703b65db54644f7b81d3dcb79bf355c1a7c28a12944e5cc6787531ccc828a"}, + {file = "tldextract-3.2.0.tar.gz", hash = "sha256:3d4b6a2105600b7d0290ea237bf30b6b0dc763e50fcbe40e849a019bd6dbcbff"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tomli = [ + {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, + {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, +] +typing-extensions = [] urllib3 = [] virtualenv = [] wrapt = [] -yarl = [] +yarl = [ + {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2a8508f7350512434e41065684076f640ecce176d262a7d54f0da41d99c5a95"}, + {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da6df107b9ccfe52d3a48165e48d72db0eca3e3029b5b8cb4fe6ee3cb870ba8b"}, + {file = "yarl-1.7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1d0894f238763717bdcfea74558c94e3bc34aeacd3351d769460c1a586a8b05"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4b95b7e00c6635a72e2d00b478e8a28bfb122dc76349a06e20792eb53a523"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c145ab54702334c42237a6c6c4cc08703b6aa9b94e2f227ceb3d477d20c36c63"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca56f002eaf7998b5fcf73b2421790da9d2586331805f38acd9997743114e98"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1d3d5ad8ea96bd6d643d80c7b8d5977b4e2fb1bab6c9da7322616fd26203d125"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:167ab7f64e409e9bdd99333fe8c67b5574a1f0495dcfd905bc7454e766729b9e"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:95a1873b6c0dd1c437fb3bb4a4aaa699a48c218ac7ca1e74b0bee0ab16c7d60d"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6152224d0a1eb254f97df3997d79dadd8bb2c1a02ef283dbb34b97d4f8492d23"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bb7d54b8f61ba6eee541fba4b83d22b8a046b4ef4d8eb7f15a7e35db2e1e245"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:9c1f083e7e71b2dd01f7cd7434a5f88c15213194df38bc29b388ccdf1492b739"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f44477ae29025d8ea87ec308539f95963ffdc31a82f42ca9deecf2d505242e72"}, + {file = "yarl-1.7.2-cp310-cp310-win32.whl", hash = "sha256:cff3ba513db55cc6a35076f32c4cdc27032bd075c9faef31fec749e64b45d26c"}, + {file = "yarl-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:c9c6d927e098c2d360695f2e9d38870b2e92e0919be07dbe339aefa32a090265"}, + {file = "yarl-1.7.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9b4c77d92d56a4c5027572752aa35082e40c561eec776048330d2907aead891d"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c01a89a44bb672c38f42b49cdb0ad667b116d731b3f4c896f72302ff77d71656"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c19324a1c5399b602f3b6e7db9478e5b1adf5cf58901996fc973fe4fccd73eed"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3abddf0b8e41445426d29f955b24aeecc83fa1072be1be4e0d194134a7d9baee"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6a1a9fe17621af43e9b9fcea8bd088ba682c8192d744b386ee3c47b56eaabb2c"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8b0915ee85150963a9504c10de4e4729ae700af11df0dc5550e6587ed7891e92"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:29e0656d5497733dcddc21797da5a2ab990c0cb9719f1f969e58a4abac66234d"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:bf19725fec28452474d9887a128e98dd67eee7b7d52e932e6949c532d820dc3b"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:d6f3d62e16c10e88d2168ba2d065aa374e3c538998ed04996cd373ff2036d64c"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ac10bbac36cd89eac19f4e51c032ba6b412b3892b685076f4acd2de18ca990aa"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:aa32aaa97d8b2ed4e54dc65d241a0da1c627454950f7d7b1f95b13985afd6c5d"}, + {file = "yarl-1.7.2-cp36-cp36m-win32.whl", hash = "sha256:87f6e082bce21464857ba58b569370e7b547d239ca22248be68ea5d6b51464a1"}, + {file = "yarl-1.7.2-cp36-cp36m-win_amd64.whl", hash = "sha256:ac35ccde589ab6a1870a484ed136d49a26bcd06b6a1c6397b1967ca13ceb3913"}, + {file = "yarl-1.7.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a467a431a0817a292121c13cbe637348b546e6ef47ca14a790aa2fa8cc93df63"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ab0c3274d0a846840bf6c27d2c60ba771a12e4d7586bf550eefc2df0b56b3b4"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d260d4dc495c05d6600264a197d9d6f7fc9347f21d2594926202fd08cf89a8ba"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc4dd8b01a8112809e6b636b00f487846956402834a7fd59d46d4f4267181c41"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c1164a2eac148d85bbdd23e07dfcc930f2e633220f3eb3c3e2a25f6148c2819e"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:67e94028817defe5e705079b10a8438b8cb56e7115fa01640e9c0bb3edf67332"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:89ccbf58e6a0ab89d487c92a490cb5660d06c3a47ca08872859672f9c511fc52"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8cce6f9fa3df25f55521fbb5c7e4a736683148bcc0c75b21863789e5185f9185"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:211fcd65c58bf250fb994b53bc45a442ddc9f441f6fec53e65de8cba48ded986"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c10ea1e80a697cf7d80d1ed414b5cb8f1eec07d618f54637067ae3c0334133c4"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:52690eb521d690ab041c3919666bea13ab9fbff80d615ec16fa81a297131276b"}, + {file = "yarl-1.7.2-cp37-cp37m-win32.whl", hash = "sha256:695ba021a9e04418507fa930d5f0704edbce47076bdcfeeaba1c83683e5649d1"}, + {file = "yarl-1.7.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c17965ff3706beedafd458c452bf15bac693ecd146a60a06a214614dc097a271"}, + {file = "yarl-1.7.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fce78593346c014d0d986b7ebc80d782b7f5e19843ca798ed62f8e3ba8728576"}, + {file = "yarl-1.7.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c2a1ac41a6aa980db03d098a5531f13985edcb451bcd9d00670b03129922cd0d"}, + {file = "yarl-1.7.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:39d5493c5ecd75c8093fa7700a2fb5c94fe28c839c8e40144b7ab7ccba6938c8"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eb6480ef366d75b54c68164094a6a560c247370a68c02dddb11f20c4c6d3c9d"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ba63585a89c9885f18331a55d25fe81dc2d82b71311ff8bd378fc8004202ff6"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e39378894ee6ae9f555ae2de332d513a5763276a9265f8e7cbaeb1b1ee74623a"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c0910c6b6c31359d2f6184828888c983d54d09d581a4a23547a35f1d0b9484b1"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6feca8b6bfb9eef6ee057628e71e1734caf520a907b6ec0d62839e8293e945c0"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8300401dc88cad23f5b4e4c1226f44a5aa696436a4026e456fe0e5d2f7f486e6"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:788713c2896f426a4e166b11f4ec538b5736294ebf7d5f654ae445fd44270832"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fd547ec596d90c8676e369dd8a581a21227fe9b4ad37d0dc7feb4ccf544c2d59"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:737e401cd0c493f7e3dd4db72aca11cfe069531c9761b8ea474926936b3c57c8"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf81561f2972fb895e7844882898bda1eef4b07b5b385bcd308d2098f1a767b"}, + {file = "yarl-1.7.2-cp38-cp38-win32.whl", hash = "sha256:ede3b46cdb719c794427dcce9d8beb4abe8b9aa1e97526cc20de9bd6583ad1ef"}, + {file = "yarl-1.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:cc8b7a7254c0fc3187d43d6cb54b5032d2365efd1df0cd1749c0c4df5f0ad45f"}, + {file = "yarl-1.7.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:580c1f15500e137a8c37053e4cbf6058944d4c114701fa59944607505c2fe3a0"}, + {file = "yarl-1.7.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ec1d9a0d7780416e657f1e405ba35ec1ba453a4f1511eb8b9fbab81cb8b3ce1"}, + {file = "yarl-1.7.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3bf8cfe8856708ede6a73907bf0501f2dc4e104085e070a41f5d88e7faf237f3"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1be4bbb3d27a4e9aa5f3df2ab61e3701ce8fcbd3e9846dbce7c033a7e8136746"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:534b047277a9a19d858cde163aba93f3e1677d5acd92f7d10ace419d478540de"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6ddcd80d79c96eb19c354d9dca95291589c5954099836b7c8d29278a7ec0bda"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9bfcd43c65fbb339dc7086b5315750efa42a34eefad0256ba114cd8ad3896f4b"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f64394bd7ceef1237cc604b5a89bf748c95982a84bcd3c4bbeb40f685c810794"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044daf3012e43d4b3538562da94a88fb12a6490652dbc29fb19adfa02cf72eac"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:368bcf400247318382cc150aaa632582d0780b28ee6053cd80268c7e72796dec"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:bab827163113177aee910adb1f48ff7af31ee0289f434f7e22d10baf624a6dfe"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0cba38120db72123db7c58322fa69e3c0efa933040ffb586c3a87c063ec7cae8"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:59218fef177296451b23214c91ea3aba7858b4ae3306dde120224cfe0f7a6ee8"}, + {file = "yarl-1.7.2-cp39-cp39-win32.whl", hash = "sha256:1edc172dcca3f11b38a9d5c7505c83c1913c0addc99cd28e993efeaafdfaa18d"}, + {file = "yarl-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:797c2c412b04403d2da075fb93c123df35239cd7b4cc4e0cd9e5839b73f52c58"}, + {file = "yarl-1.7.2.tar.gz", hash = "sha256:45399b46d60c253327a460e99856752009fcee5f5d3c80b2f7c0cae1c38d56dd"}, +] diff --git a/pyproject.toml b/pyproject.toml index 241ec3e99f..975b9b9958 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ rapidfuzz = "2.0.7" lxml = "4.9.1" markdownify = "0.6.1" more_itertools = "8.12.0" +pydantic = "~=1.9" python-dateutil = "2.8.2" python-frontmatter = "1.0.0" pyyaml = "6.0" From 9670acce01bc81ca608aa8881f9394577b81ca99 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 5 Mar 2022 18:38:47 +0200 Subject: [PATCH 010/132] Add system description commands The system is fairly complex, and to avoid having to delve into the source code, this commit adds several commands to show descriptions for various components of the system: - Filters - Filter lists - Settings (both from the DB schema and filter-type-specific) The names that can be described are the ones which are actually being used in the system. So if no tokens list was loaded, it can't be described even if there is an implementation for it. If no name is relayed to the commands, the list of options will be shown instead. --- bot/exts/filtering/_filter_lists/domain.py | 18 ++- bot/exts/filtering/_filter_lists/extension.py | 19 ++- .../filtering/_filter_lists/filter_list.py | 5 + bot/exts/filtering/_filter_lists/invite.py | 22 ++- bot/exts/filtering/_filter_lists/token.py | 19 ++- bot/exts/filtering/_filters/domain.py | 19 ++- bot/exts/filtering/_filters/extension.py | 8 +- bot/exts/filtering/_filters/filter.py | 13 +- bot/exts/filtering/_filters/invite.py | 8 +- bot/exts/filtering/_filters/token.py | 6 +- .../filtering/_settings_types/bypass_roles.py | 1 + .../_settings_types/channel_scope.py | 10 ++ .../_settings_types/delete_messages.py | 1 + bot/exts/filtering/_settings_types/enabled.py | 1 + .../filtering/_settings_types/filter_dm.py | 1 + .../infraction_and_notification.py | 13 ++ bot/exts/filtering/_settings_types/ping.py | 10 ++ .../filtering/_settings_types/send_alert.py | 2 +- .../_settings_types/settings_entry.py | 3 + bot/exts/filtering/filtering.py | 130 ++++++++++++++++++ 20 files changed, 287 insertions(+), 22 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index c481643694..05c520ce22 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -4,11 +4,12 @@ import typing from functools import reduce from operator import or_ -from typing import Optional +from typing import Optional, Type from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType from bot.exts.filtering._filters.domain import DomainFilter +from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._utils import clean_input @@ -19,7 +20,15 @@ class DomainsList(FilterList): - """A list of filters, each looking for a specific domain given by URL.""" + """ + A list of filters, each looking for a specific domain given by URL. + + The blacklist defaults dictate what happens by default when a filter is matched, and can be overridden by + individual filters. + + Domains are found by looking for a URL schema (http or https). + Filters will also trigger for subdomains unless set otherwise. + """ name = "domain" @@ -27,6 +36,11 @@ def __init__(self, filtering_cog: Filtering): super().__init__(DomainFilter) filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + @property + def filter_types(self) -> set[Type[Filter]]: + """Return the types of filters used by this list.""" + return {DomainFilter} + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" text = ctx.content diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index ceb8bb9580..b70ab67729 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -2,13 +2,14 @@ import typing from os.path import splitext -from typing import Optional +from typing import Optional, Type import bot from bot.constants import Channels, URLs from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType from bot.exts.filtering._filters.extension import ExtensionFilter +from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings if typing.TYPE_CHECKING: @@ -34,7 +35,16 @@ class ExtensionsList(FilterList): - """A list of filters, each looking for an attachment with a specific extension.""" + """ + A list of filters, each looking for a file attachment with a specific extension. + + If an extension is not explicitly allowed, it will be blocked. + + Whitelist defaults dictate what happens when an extension is *not* explicitly allowed, + and whitelist filters overrides have no effect. + + Items should be added as file extensions preceded by a dot. + """ name = "extension" @@ -43,6 +53,11 @@ def __init__(self, filtering_cog: Filtering): filtering_cog.subscribe(self, Event.MESSAGE) self._whitelisted_description = None + @property + def filter_types(self) -> set[Type[Filter]]: + """Return the types of filters used by this list.""" + return {ExtensionFilter} + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" # Return early if the message doesn't have attachments. diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 5fc9925974..60c884a040 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -63,6 +63,11 @@ def add_list(self, list_data: Dict) -> None: log.warning(e) self.filter_lists[list_type] = filters + @property + @abstractmethod + def filter_types(self) -> set[Type[Filter]]: + """Return the types of filters used by this list.""" + @abstractmethod async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index cadd82d0c1..c79cd9b511 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -3,7 +3,7 @@ import typing from functools import reduce from operator import or_ -from typing import Optional +from typing import Optional, Type from botcore.utils.regex import DISCORD_INVITE from discord import Embed, Invite @@ -12,6 +12,7 @@ import bot from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType +from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._filters.invite import InviteFilter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._utils import clean_input @@ -21,7 +22,19 @@ class InviteList(FilterList): - """A list of filters, each looking for guild invites to a specific guild.""" + """ + A list of filters, each looking for guild invites to a specific guild. + + If the invite is not whitelisted, it will be blocked. Partnered and verified servers are allowed unless blacklisted. + + Whitelist defaults dictate what happens when an invite is *not* explicitly allowed, + and whitelist filters overrides have no effect. + + Blacklist defaults dictate what happens by default when an explicitly blocked invite is found. + + Items in the list are added through invites for the purpose of fetching the guild info. + Items are stored as guild IDs, guild invites are *not* stored. + """ name = "invite" @@ -29,6 +42,11 @@ def __init__(self, filtering_cog: Filtering): super().__init__(InviteFilter) filtering_cog.subscribe(self, Event.MESSAGE) + @property + def filter_types(self) -> set[Type[Filter]]: + """Return the types of filters used by this list.""" + return {InviteFilter} + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" _, failed = self.defaults[ListType.ALLOW]["validations"].evaluate(ctx) diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index c232b55e5b..5be3fd0e8e 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -4,10 +4,11 @@ import typing from functools import reduce from operator import or_ -from typing import Optional +from typing import Optional, Type from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType +from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._filters.token import TokenFilter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._utils import clean_input @@ -19,7 +20,16 @@ class TokensList(FilterList): - """A list of filters, each looking for a specific token given by regex.""" + """ + A list of filters, each looking for a specific token in the given content given as regex. + + The blacklist defaults dictate what happens by default when a filter is matched, and can be overridden by + individual filters. + + Usually, if blocking literal strings, the literals themselves can be specified as the filter's value. + But since this is a list of regex patterns, be careful of the items added. For example, a dot needs to be escaped + to function as a literal dot. + """ name = "token" @@ -27,6 +37,11 @@ def __init__(self, filtering_cog: Filtering): super().__init__(TokenFilter) filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + @property + def filter_types(self) -> set[Type[Filter]]: + """Return the types of filters used by this list.""" + return {TokenFilter} + async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" text = ctx.content diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 1511d6a5cc..9f5f974134 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -1,26 +1,33 @@ -from typing import Optional +from typing import ClassVar, Optional import tldextract from pydantic import BaseModel from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter -from bot.exts.filtering._settings import ActionSettings class ExtraDomainSettings(BaseModel): """Extra settings for how domains should be matched in a message.""" + exact_description: ClassVar[str] = ( + "A boolean. If True, will match the filter content exactly, and won't trigger for subdomains and subpaths." + ) + # whether to match the filter content exactly, or to trigger for subdomains and subpaths as well. exact: Optional[bool] = False class DomainFilter(Filter): - """A filter which looks for a specific domain given by URL.""" + """ + A filter which looks for a specific domain given by URL. + + The schema (http, https) does not need to be included in the filter. + Will also match subdomains unless set otherwise. + """ - def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): - super().__init__(filter_data, action_defaults) - self.extra_fields = ExtraDomainSettings.parse_raw(self.extra_fields) + name = "domain" + extra_fields_type = ExtraDomainSettings def triggered_on(self, ctx: FilterContext) -> bool: """Searches for a domain within a given context.""" diff --git a/bot/exts/filtering/_filters/extension.py b/bot/exts/filtering/_filters/extension.py index 85bfd05b2b..1a2ab86174 100644 --- a/bot/exts/filtering/_filters/extension.py +++ b/bot/exts/filtering/_filters/extension.py @@ -3,7 +3,13 @@ class ExtensionFilter(Filter): - """A filter which looks for a specific attachment extension in messages.""" + """ + A filter which looks for a specific attachment extension in messages. + + The filter stores the extension preceded by a dot. + """ + + name = "extension" def triggered_on(self, ctx: FilterContext) -> bool: """Searches for an attachment extension in the context content, given as a set of extensions.""" diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index b4beb8386d..d27b3dae33 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,11 +1,12 @@ -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Optional from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings import ActionSettings, create_settings +from bot.exts.filtering._utils import FieldRequiring -class Filter(ABC): +class Filter(FieldRequiring): """ A class representing a filter. @@ -13,6 +14,12 @@ class Filter(ABC): and defines what action should be performed if it is triggered. """ + # Each subclass must define a name which will be used to fetch its description. + # Names must be unique across all types of filters. + name = FieldRequiring.MUST_SET_UNIQUE + # If a subclass uses extra fields, it should assign the pydantic model type to this variable. + extra_fields_type = None + def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): self.id = filter_data["id"] self.content = filter_data["content"] @@ -23,6 +30,8 @@ def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] elif action_defaults: self.actions.fallback_to(action_defaults) self.extra_fields = filter_data["additional_field"] or "{}" # noqa: P103 + if self.extra_fields_type: + self.extra_fields = self.extra_fields_type.parse_raw(self.extra_fields) @abstractmethod def triggered_on(self, ctx: FilterContext) -> bool: diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py index afe4fdd945..e5b68258cf 100644 --- a/bot/exts/filtering/_filters/invite.py +++ b/bot/exts/filtering/_filters/invite.py @@ -6,7 +6,13 @@ class InviteFilter(Filter): - """A filter which looks for invites to a specific guild in messages.""" + """ + A filter which looks for invites to a specific guild in messages. + + The filter stores the guild ID which is allowed or denied. + """ + + name = "invite" def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): super().__init__(filter_data, action_defaults) diff --git a/bot/exts/filtering/_filters/token.py b/bot/exts/filtering/_filters/token.py index 07590c54b3..c955b269ba 100644 --- a/bot/exts/filtering/_filters/token.py +++ b/bot/exts/filtering/_filters/token.py @@ -1,12 +1,14 @@ import re -from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filters.filter import Filter class TokenFilter(Filter): """A filter which looks for a specific token given by regex.""" + name = "token" + def triggered_on(self, ctx: FilterContext) -> bool: """Searches for a regex pattern within a given context.""" pattern = self.content @@ -16,5 +18,3 @@ def triggered_on(self, ctx: FilterContext) -> bool: ctx.matches.append(match[0]) return True return False - - diff --git a/bot/exts/filtering/_settings_types/bypass_roles.py b/bot/exts/filtering/_settings_types/bypass_roles.py index bfc4a30fd0..290ea53c13 100644 --- a/bot/exts/filtering/_settings_types/bypass_roles.py +++ b/bot/exts/filtering/_settings_types/bypass_roles.py @@ -10,6 +10,7 @@ class RoleBypass(ValidationEntry): """A setting entry which tells whether the roles the member has allow them to bypass the filter.""" name = "bypass_roles" + description = "A list of role IDs or role names. Users with these roles will not trigger the filter." def __init__(self, entry_data: Any): super().__init__(entry_data) diff --git a/bot/exts/filtering/_settings_types/channel_scope.py b/bot/exts/filtering/_settings_types/channel_scope.py index 63da6c7e50..3a95834b33 100644 --- a/bot/exts/filtering/_settings_types/channel_scope.py +++ b/bot/exts/filtering/_settings_types/channel_scope.py @@ -15,6 +15,16 @@ class ChannelScope(ValidationEntry): """A setting entry which tells whether the filter was invoked in a whitelisted channel or category.""" name = "channel_scope" + description = { + "disabled_channels": "A list of channel IDs or channel names. The filter will not trigger in these channels.", + "disabled_categories": ( + "A list of category IDs or category names. The filter will not trigger in these categories." + ), + "enabled_channels": ( + "A list of channel IDs or channel names. " + "The filter can trigger in these channels even if the category is disabled." + ) + } def __init__(self, entry_data: Any): super().__init__(entry_data) diff --git a/bot/exts/filtering/_settings_types/delete_messages.py b/bot/exts/filtering/_settings_types/delete_messages.py index ad715f04c9..8de58f8046 100644 --- a/bot/exts/filtering/_settings_types/delete_messages.py +++ b/bot/exts/filtering/_settings_types/delete_messages.py @@ -11,6 +11,7 @@ class DeleteMessages(ActionEntry): """A setting entry which tells whether to delete the offending message(s).""" name = "delete_messages" + description = "A boolean field. If True, the filter being triggered will cause the offending message to be deleted." def __init__(self, entry_data: Any): super().__init__(entry_data) diff --git a/bot/exts/filtering/_settings_types/enabled.py b/bot/exts/filtering/_settings_types/enabled.py index 553dccc9cf..081ae02b0c 100644 --- a/bot/exts/filtering/_settings_types/enabled.py +++ b/bot/exts/filtering/_settings_types/enabled.py @@ -8,6 +8,7 @@ class Enabled(ValidationEntry): """A setting entry which tells whether the filter is enabled.""" name = "enabled" + description = "A boolean field. Setting it to False allows disabling the filter without deleting it entirely." def __init__(self, entry_data: Any): super().__init__(entry_data) diff --git a/bot/exts/filtering/_settings_types/filter_dm.py b/bot/exts/filtering/_settings_types/filter_dm.py index 54f19e4d17..676e04aa9f 100644 --- a/bot/exts/filtering/_settings_types/filter_dm.py +++ b/bot/exts/filtering/_settings_types/filter_dm.py @@ -8,6 +8,7 @@ class FilterDM(ValidationEntry): """A setting entry which tells whether to apply the filter to DMs.""" name = "filter_dm" + description = "A boolean field. If True, the filter can trigger for messages sent to the bot in DMs." def __init__(self, entry_data: Any): super().__init__(entry_data) diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index 82e2ff6d65..03574049a8 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -46,6 +46,19 @@ class InfractionAndNotification(ActionEntry): """ name = "infraction_and_notification" + description = { + "infraction_type": ( + "The type of infraction to issue when the filter triggers, or 'NONE'. " + "If two infractions are triggered for the same message, " + "the harsher one will be applied (by type or duration). " + "Superstars will be triggered even if there is a harsher infraction.\n\n" + "Valid infraction types in order of harshness: " + ) + ", ".join(infraction.name for infraction in Infraction), + "infraction_duration": "How long the infraction should last for in seconds, or 'None' for permanent.", + "infraction_reason": "The reason delivered with the infraction.", + "dm_content": "The contents of a message to be DMed to the offending user.", + "dm_embed": "The contents of the embed to be DMed to the offending user." + } def __init__(self, entry_data: Any): super().__init__(entry_data) diff --git a/bot/exts/filtering/_settings_types/ping.py b/bot/exts/filtering/_settings_types/ping.py index 0f9a014c4e..9d3bef5622 100644 --- a/bot/exts/filtering/_settings_types/ping.py +++ b/bot/exts/filtering/_settings_types/ping.py @@ -11,6 +11,16 @@ class Ping(ActionEntry): """A setting entry which adds the appropriate pings to the alert.""" name = "mentions" + description = { + "ping_type": ( + "A list of role IDs/role names/user IDs/user names/here/everyone. " + "If a mod-alert is generated for a filter triggered in a public channel, these will be pinged." + ), + "dm_ping_type": ( + "A list of role IDs/role names/user IDs/user names/here/everyone. " + "If a mod-alert is generated for a filter triggered in DMs, these will be pinged." + ) + } def __init__(self, entry_data: Any): super().__init__(entry_data) diff --git a/bot/exts/filtering/_settings_types/send_alert.py b/bot/exts/filtering/_settings_types/send_alert.py index e332494eb5..6429b99ac9 100644 --- a/bot/exts/filtering/_settings_types/send_alert.py +++ b/bot/exts/filtering/_settings_types/send_alert.py @@ -8,6 +8,7 @@ class SendAlert(ActionEntry): """A setting entry which tells whether to send an alert message.""" name = "send_alert" + description = "A boolean field. If all filters triggered set this to False, no mod-alert will be created." def __init__(self, entry_data: Any): super().__init__(entry_data) @@ -23,4 +24,3 @@ def __or__(self, other: ActionEntry): return NotImplemented return SendAlert(self.send_alert or other.send_alert) - diff --git a/bot/exts/filtering/_settings_types/settings_entry.py b/bot/exts/filtering/_settings_types/settings_entry.py index b0d54fac37..c3a1a8a070 100644 --- a/bot/exts/filtering/_settings_types/settings_entry.py +++ b/bot/exts/filtering/_settings_types/settings_entry.py @@ -17,6 +17,9 @@ class SettingsEntry(FieldRequiring): # Each subclass must define a name matching the entry name we're expecting to receive from the database. # Names must be unique across all filter lists. name = FieldRequiring.MUST_SET_UNIQUE + # Each subclass must define a description of what it does. If the data an entry type receives is comprised of + # several DB fields, the value should a dictionary of field names and their descriptions. + description = FieldRequiring.MUST_SET @abstractmethod def __init__(self, entry_data: Any): diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 2e5cca5fae..2cfb456567 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1,4 +1,5 @@ import operator +import re from collections import defaultdict from functools import reduce from typing import Optional @@ -33,6 +34,10 @@ def __init__(self, bot: Bot): self._subscriptions: defaultdict[Event, list[FilterList]] = defaultdict(list) self.webhook = None + self.loaded_settings = {} + self.loaded_filters = {} + self.loaded_filter_settings = {} + async def cog_load(self) -> None: """ Fetch the filter data from the API, parse it, and load it to the appropriate data structures. @@ -61,6 +66,8 @@ async def cog_load(self) -> None: except HTTPException: log.error(f"Failed to fetch incidents webhook with ID `{Webhooks.incidents}`.") + self.collect_loaded_types() + def subscribe(self, filter_list: FilterList, *events: Event) -> None: """ Subscribe a filter list to the given events. @@ -78,6 +85,49 @@ def subscribe(self, filter_list: FilterList, *events: Event) -> None: if filter_list not in self._subscriptions[event]: self._subscriptions[event].append(filter_list) + def collect_loaded_types(self) -> None: + """ + Go over the classes used in initialization and collect them to dictionaries. + + The information that is collected is about the types actually used to load the API response, not all types + available in the filtering extension. + """ + # Get the filter types used by each filter list. + for filter_list in self.filter_lists.values(): + self.loaded_filters.update({filter_type.name: filter_type for filter_type in filter_list.filter_types}) + + # Get the setting types used by each filter list. + if self.filter_lists: + # Any filter list has the fields for all settings in the DB schema, so picking any one of them is enough. + list_defaults = list(list(self.filter_lists.values())[0].defaults.values())[0] + settings_types = set() + # The settings are split between actions and validations. + settings_types.update(type(setting) for _, setting in list_defaults["actions"].items()) + settings_types.update(type(setting) for _, setting in list_defaults["validations"].items()) + for setting_type in settings_types: + # The description should be either a string or a dictionary. + if isinstance(setting_type.description, str): + # If it's a string, then the setting matches a single field in the DB, + # and its name is the setting type's name attribute. + self.loaded_settings[setting_type.name] = setting_type.description, setting_type + else: + # Otherwise, the setting type works with compound settings. + self.loaded_settings.update({ + subsetting: (description, setting_type) + for subsetting, description in setting_type.description.items() + }) + + # Get the settings per filter as well. + for filter_name, filter_type in self.loaded_filters.items(): + extra_fields_type = filter_type.extra_fields_type + if not extra_fields_type: + continue + # A class var with a `_description` suffix is expected per field name. + self.loaded_filter_settings[filter_name] = { + field_name: (getattr(extra_fields_type, f"{field_name}_description", ""), extra_fields_type) + for field_name in extra_fields_type.__fields__ + } + async def cog_check(self, ctx: Context) -> bool: """Only allow moderators to invoke the commands in this cog.""" return await has_any_role(*MODERATION_ROLES).predicate(ctx) @@ -174,6 +224,86 @@ async def f_list( await self._send_list(ctx, list_name, list_type) + @filter.command(name="describe", aliases=("explain", "manual")) + async def f_describe(self, ctx: Context, filter_name: Optional[str]) -> None: + """Show a description of the specified filter, or a list of possible values if no name is specified.""" + if not filter_name: + embed = Embed(description="\n".join(self.loaded_filters)) + embed.set_author(name="List of filter names") + else: + filter_type = self.loaded_filters.get(filter_name) + if not filter_type: + filter_type = self.loaded_filters.get(filter_name[:-1]) # A plural form or a typo. + if not filter_type: + await ctx.send(f":x: There's no filter type named {filter_name!r}.") + return + # Use the class's docstring, and ignore single newlines. + embed = Embed(description=re.sub(r"(? None: + """Group for settings-related commands.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @setting.command(name="describe", aliases=("explain", "manual")) + async def s_describe(self, ctx: Context, setting_name: Optional[str]) -> None: + """Show a description of the specified setting, or a list of possible settings if no name is specified.""" + if not setting_name: + settings_list = list(self.loaded_settings) + for filter_name, filter_settings in self.loaded_filter_settings.items(): + settings_list.extend(f"{filter_name}/{setting}" for setting in filter_settings) + embed = Embed(description="\n".join(settings_list)) + embed.set_author(name="List of setting names") + else: + # The setting is either in a SettingsEntry subclass, or a pydantic model. + setting_data = self.loaded_settings.get(setting_name) + description = None + if setting_data: + description = setting_data[0] + elif "/" in setting_name: # It's a filter specific setting. + filter_name, filter_setting_name = setting_name.split("/", maxsplit=1) + if filter_name in self.loaded_filter_settings: + if filter_setting_name in self.loaded_filter_settings[filter_name]: + description = self.loaded_filter_settings[filter_name][filter_setting_name][0] + if description is None: + await ctx.send(f":x: There's no setting type named {setting_name!r}.") + return + embed = Embed(description=description) + embed.set_author(name=f"Description of the {setting_name} setting") + embed.colour = Colour.blue() + await ctx.send(embed=embed) + + # endregion + # region: filterlist group + + @commands.group(aliases=("fl",)) + async def filterlist(self, ctx: Context) -> None: + """Group for managing filter lists.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @filterlist.command(name="describe", aliases=("explain", "manual")) + async def fl_describe(self, ctx: Context, filterlist_name: Optional[str]) -> None: + """Show a description of the specified filter list, or a list of possible values if no name is specified.""" + if not filterlist_name: + embed = Embed(description="\n".join(self.filter_lists)) + embed.set_author(name="List of filter lists names") + else: + try: + filter_list = self._get_list_by_name(filterlist_name) + except BadArgument as e: + await ctx.send(f":x: {e}") + return + # Use the class's docstring, and ignore single newlines. + embed = Embed(description=re.sub(r"(? Date: Tue, 8 Mar 2022 20:51:06 +0200 Subject: [PATCH 011/132] Add settings display for individual filters and filter lists The filterlist describe command is changed to include the default settings it contains. The filters group can now take a filter ID, and it will display embed detailing the filter data and settings. The mechanic is similar to how individual infractions can be displayed with `!infraction `. - To be able to quickly find whether the filter with the provided ID is in a specific list, the data structure was changed to a dictionary. - To be able to mark which settings have the default values and which are overrides, resolving the full actions of a filter is deferred to when the filter is actually triggered. This wasn't possible in the beginning of development, but now that each filterlist can resolve the action to be taken with its own internal logic, it is. - Some attribute names of SettingsEntry subclasses were changed to match the name under which they're stored in the DB. This allows displaying the settings with the names that need to be used to edit them. - Each filterlist now contains all settings, even if they're empty, so that they can be displayed. While this is slightly less efficient, I considered it too negligible to make displaying the settings messier than it already is. - Some additional refactoring in the cog was done to avoid code repetition. --- bot/exts/filtering/_filter_lists/domain.py | 8 +- bot/exts/filtering/_filter_lists/extension.py | 4 +- .../filtering/_filter_lists/filter_list.py | 18 +- bot/exts/filtering/_filter_lists/invite.py | 14 +- bot/exts/filtering/_filter_lists/token.py | 8 +- bot/exts/filtering/_filters/filter.py | 9 +- bot/exts/filtering/_filters/invite.py | 7 +- bot/exts/filtering/_settings.py | 33 +-- .../filtering/_settings_types/bypass_roles.py | 8 +- .../filtering/_settings_types/filter_dm.py | 4 +- .../infraction_and_notification.py | 5 + bot/exts/filtering/_settings_types/ping.py | 15 +- .../_settings_types/settings_entry.py | 4 +- bot/exts/filtering/_utils.py | 20 +- bot/exts/filtering/filtering.py | 199 +++++++++++++----- 15 files changed, 248 insertions(+), 108 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index 05c520ce22..7f92b62e87 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -58,7 +58,13 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings actions = None message = "" if triggers: - actions = reduce(or_, (filter_.actions for filter_ in triggers)) + action_defaults = self.defaults[ListType.DENY]["actions"] + actions = reduce( + or_, + (filter_.actions.fallback_to(action_defaults) if filter_.actions else action_defaults + for filter_ in triggers + ) + ) if len(triggers) == 1: message = f"#{triggers[0].id} (`{triggers[0].content}`)" if triggers[0].description: diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index b70ab67729..2447bebde7 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -73,7 +73,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings (splitext(attachment.filename.lower())[1], attachment.filename) for attachment in ctx.message.attachments } new_ctx = ctx.replace(content={ext for ext, _ in all_ext}) # And prepare the context for the filters to read. - triggered = [filter_ for filter_ in self.filter_lists[ListType.ALLOW] if filter_.triggered_on(new_ctx)] + triggered = [filter_ for filter_ in self.filter_lists[ListType.ALLOW].values() if filter_.triggered_on(new_ctx)] allowed_ext = {filter_.content for filter_ in triggered} # Get the extensions in the message that are allowed. # See if there are any extensions left which aren't allowed. @@ -97,7 +97,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings meta_channel = bot.instance.get_channel(Channels.meta) if not self._whitelisted_description: self._whitelisted_description = ', '.join( - filter_.content for filter_ in self.filter_lists[ListType.ALLOW] + filter_.content for filter_ in self.filter_lists[ListType.ALLOW].values() ) ctx.dm_embed = DISALLOWED_EMBED_DESCRIPTION.format( joined_whitelist=self._whitelisted_description, diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 60c884a040..3b5138fe44 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,6 +1,6 @@ from abc import abstractmethod from enum import Enum -from typing import Dict, List, Optional, Type +from typing import Optional, Type from discord.ext.commands import BadArgument @@ -44,21 +44,21 @@ class FilterList(FieldRequiring): name = FieldRequiring.MUST_SET_UNIQUE def __init__(self, filter_type: Type[Filter]): - self.filter_lists: dict[ListType, list[Filter]] = {} + self.filter_lists: dict[ListType, dict[int, Filter]] = {} self.defaults = {} self.filter_type = filter_type - def add_list(self, list_data: Dict) -> None: + def add_list(self, list_data: dict) -> None: """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" - actions, validations = create_settings(list_data["settings"]) + actions, validations = create_settings(list_data["settings"], keep_empty=True) list_type = ListType(list_data["list_type"]) self.defaults[list_type] = {"actions": actions, "validations": validations} - filters = [] + filters = {} for filter_data in list_data["filters"]: try: - filters.append(self.filter_type(filter_data, actions)) + filters[filter_data["id"]] = self.filter_type(filter_data) except TypeError as e: log.warning(e) self.filter_lists[list_type] = filters @@ -73,7 +73,9 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" @staticmethod - def filter_list_result(ctx: FilterContext, filters: List[Filter], defaults: ValidationSettings) -> list[Filter]: + def filter_list_result( + ctx: FilterContext, filters: dict[int, Filter], defaults: ValidationSettings + ) -> list[Filter]: """ Sift through the list of filters, and return only the ones which apply to the given context. @@ -91,7 +93,7 @@ def filter_list_result(ctx: FilterContext, filters: List[Filter], defaults: Vali default_answer = not bool(failed_by_default) relevant_filters = [] - for filter_ in filters: + for filter_ in filters.values(): if not filter_.validations: if default_answer and filter_.triggered_on(ctx): relevant_filters.append(filter_) diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index c79cd9b511..4e8d74d8a3 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -84,7 +84,9 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings # Add the disallowed by default unless they're whitelisted. guilds_for_inspection = {invite.guild.id for invite in denied_by_default.values()} new_ctx = ctx.replace(content=guilds_for_inspection) - allowed = {filter_.content for filter_ in self.filter_lists[ListType.ALLOW] if filter_.triggered_on(new_ctx)} + allowed = { + filter_.content for filter_ in self.filter_lists[ListType.ALLOW].values() if filter_.triggered_on(new_ctx) + } disallowed_invites.update({ invite_code: invite for invite_code, invite in denied_by_default.items() if invite.guild.id not in allowed }) @@ -105,7 +107,15 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings actions = None if len(disallowed_invites) > len(triggered): # There are invites which weren't allowed but aren't blacklisted. - actions = reduce(or_, (filter_.actions for filter_ in triggered), self.defaults[ListType.ALLOW]["actions"]) + deny_defaults = self.defaults[ListType.DENY]["actions"] + actions = reduce( + or_, + ( + filter_.actions.fallback_to(deny_defaults) if filter_.actions else deny_defaults + for filter_ in triggered + ), + self.defaults[ListType.ALLOW]["actions"] + ) elif triggered: actions = reduce(or_, (filter_.actions for filter_ in triggered)) ctx.matches += {match[0] for match in matches if match.group("invite") in disallowed_invites} diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index 5be3fd0e8e..c989b06b9e 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -58,7 +58,13 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings actions = None message = "" if triggers: - actions = reduce(or_, (filter_.actions for filter_ in triggers)) + action_defaults = self.defaults[ListType.DENY]["actions"] + actions = reduce( + or_, + (filter_.actions.fallback_to(action_defaults) if filter_.actions else action_defaults + for filter_ in triggers + ) + ) if len(triggers) == 1: message = f"#{triggers[0].id} (`{triggers[0].content}`)" if triggers[0].description: diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index d27b3dae33..da149dce60 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,8 +1,7 @@ from abc import abstractmethod -from typing import Optional from bot.exts.filtering._filter_context import FilterContext -from bot.exts.filtering._settings import ActionSettings, create_settings +from bot.exts.filtering._settings import create_settings from bot.exts.filtering._utils import FieldRequiring @@ -20,15 +19,11 @@ class Filter(FieldRequiring): # If a subclass uses extra fields, it should assign the pydantic model type to this variable. extra_fields_type = None - def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): + def __init__(self, filter_data: dict): self.id = filter_data["id"] self.content = filter_data["content"] self.description = filter_data["description"] self.actions, self.validations = create_settings(filter_data["settings"]) - if not self.actions: - self.actions = action_defaults - elif action_defaults: - self.actions.fallback_to(action_defaults) self.extra_fields = filter_data["additional_field"] or "{}" # noqa: P103 if self.extra_fields_type: self.extra_fields = self.extra_fields_type.parse_raw(self.extra_fields) diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py index e5b68258cf..5a99248331 100644 --- a/bot/exts/filtering/_filters/invite.py +++ b/bot/exts/filtering/_filters/invite.py @@ -1,8 +1,5 @@ -from typing import Optional - from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter -from bot.exts.filtering._settings import ActionSettings class InviteFilter(Filter): @@ -14,8 +11,8 @@ class InviteFilter(Filter): name = "invite" - def __init__(self, filter_data: dict, action_defaults: Optional[ActionSettings] = None): - super().__init__(filter_data, action_defaults) + def __init__(self, filter_data: dict): + super().__init__(filter_data) self.content = int(self.content) def triggered_on(self, ctx: FilterContext) -> bool: diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index b53400b78c..f88b26ee32 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -16,7 +16,9 @@ _already_warned: set[str] = set() -def create_settings(settings_data: dict) -> tuple[Optional[ActionSettings], Optional[ValidationSettings]]: +def create_settings( + settings_data: dict, *, keep_empty: bool = False +) -> tuple[Optional[ActionSettings], Optional[ValidationSettings]]: """ Create and return instances of the Settings subclasses from the given data. @@ -34,7 +36,10 @@ def create_settings(settings_data: dict) -> tuple[Optional[ActionSettings], Opti f"A setting named {entry_name} was loaded from the database, but no matching class." ) _already_warned.add(entry_name) - return ActionSettings.create(action_data), ValidationSettings.create(validation_data) + return ( + ActionSettings.create(action_data, keep_empty=keep_empty), + ValidationSettings.create(validation_data, keep_empty=keep_empty) + ) class Settings(FieldRequiring): @@ -54,7 +59,7 @@ class Settings(FieldRequiring): _already_warned: set[str] = set() @abstractmethod - def __init__(self, settings_data: dict): + def __init__(self, settings_data: dict, *, keep_empty: bool = False): self._entries: dict[str, Settings.entry_type] = {} entry_classes = settings_types.get(self.entry_type.__name__) @@ -70,7 +75,7 @@ def __init__(self, settings_data: dict): self._already_warned.add(entry_name) else: try: - new_entry = entry_cls.create(entry_data) + new_entry = entry_cls.create(entry_data, keep_empty=keep_empty) if new_entry: self._entries[entry_name] = new_entry except TypeError as e: @@ -103,17 +108,17 @@ def get(self, key: str, default: Optional[Any] = None) -> entry_type: return self._entries.get(key, default) @classmethod - def create(cls, settings_data: dict) -> Optional[Settings]: + def create(cls, settings_data: dict, *, keep_empty: bool = False) -> Optional[Settings]: """ Returns a Settings object from `settings_data` if it holds any value, None otherwise. Use this method to create Settings objects instead of the init. The None value is significant for how a filter list iterates over its filters. """ - settings = cls(settings_data) + settings = cls(settings_data, keep_empty=keep_empty) # If an entry doesn't hold any values, its `create` method will return None. # If all entries are None, then the settings object holds no values. - if not any(settings._entries.values()): + if not keep_empty and not any(settings._entries.values()): return None return settings @@ -129,8 +134,8 @@ class ValidationSettings(Settings): entry_type = ValidationEntry - def __init__(self, settings_data: dict): - super().__init__(settings_data) + def __init__(self, settings_data: dict, *, keep_empty: bool = False): + super().__init__(settings_data, keep_empty=keep_empty) def evaluate(self, ctx: FilterContext) -> tuple[set[str], set[str]]: """Evaluates for each setting whether the context is relevant to the filter.""" @@ -158,8 +163,8 @@ class ActionSettings(Settings): entry_type = ActionEntry - def __init__(self, settings_data: dict): - super().__init__(settings_data) + def __init__(self, settings_data: dict, *, keep_empty: bool = False): + super().__init__(settings_data, keep_empty=keep_empty) def __or__(self, other: ActionSettings) -> ActionSettings: """Combine the entries of two collections of settings into a new ActionsSettings.""" @@ -183,8 +188,10 @@ async def action(self, ctx: FilterContext) -> None: for entry in self._entries.values(): await entry.action(ctx) - def fallback_to(self, fallback: ActionSettings) -> None: + def fallback_to(self, fallback: ActionSettings) -> ActionSettings: """Fill in missing entries from `fallback`.""" + new_actions = self.copy() for entry_name, entry_value in fallback.items(): if entry_name not in self._entries: - self._entries[entry_name] = entry_value + new_actions._entries[entry_name] = entry_value + return new_actions diff --git a/bot/exts/filtering/_settings_types/bypass_roles.py b/bot/exts/filtering/_settings_types/bypass_roles.py index 290ea53c13..e183e0b425 100644 --- a/bot/exts/filtering/_settings_types/bypass_roles.py +++ b/bot/exts/filtering/_settings_types/bypass_roles.py @@ -14,18 +14,18 @@ class RoleBypass(ValidationEntry): def __init__(self, entry_data: Any): super().__init__(entry_data) - self.roles = set() + self.bypass_roles = set() for role in entry_data: if role.isdigit(): - self.roles.add(int(role)) + self.bypass_roles.add(int(role)) else: - self.roles.add(role) + self.bypass_roles.add(role) def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered on this user given their roles.""" if not isinstance(ctx.author, Member): return True return all( - member_role.id not in self.roles and member_role.name not in self.roles + member_role.id not in self.bypass_roles and member_role.name not in self.bypass_roles for member_role in ctx.author.roles ) diff --git a/bot/exts/filtering/_settings_types/filter_dm.py b/bot/exts/filtering/_settings_types/filter_dm.py index 676e04aa9f..1405a636f3 100644 --- a/bot/exts/filtering/_settings_types/filter_dm.py +++ b/bot/exts/filtering/_settings_types/filter_dm.py @@ -12,8 +12,8 @@ class FilterDM(ValidationEntry): def __init__(self, entry_data: Any): super().__init__(entry_data) - self.apply_in_dm = entry_data + self.filter_dm = entry_data def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered even if it was triggered in DMs.""" - return hasattr(ctx.channel, "guild") or self.apply_in_dm + return hasattr(ctx.channel, "guild") or self.filter_dm diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index 03574049a8..4fae09f235 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -34,6 +34,11 @@ def __bool__(self) -> bool: """ return self != Infraction.NONE + def __str__(self) -> str: + if self == Infraction.NONE: + return "" + return self.name + superstar = namedtuple("superstar", ["reason", "duration"]) diff --git a/bot/exts/filtering/_settings_types/ping.py b/bot/exts/filtering/_settings_types/ping.py index 9d3bef5622..1e00676905 100644 --- a/bot/exts/filtering/_settings_types/ping.py +++ b/bot/exts/filtering/_settings_types/ping.py @@ -12,11 +12,11 @@ class Ping(ActionEntry): name = "mentions" description = { - "ping_type": ( + "guild_pings": ( "A list of role IDs/role names/user IDs/user names/here/everyone. " "If a mod-alert is generated for a filter triggered in a public channel, these will be pinged." ), - "dm_ping_type": ( + "dm_pings": ( "A list of role IDs/role names/user IDs/user names/here/everyone. " "If a mod-alert is generated for a filter triggered in DMs, these will be pinged." ) @@ -24,12 +24,13 @@ class Ping(ActionEntry): def __init__(self, entry_data: Any): super().__init__(entry_data) - self.guild_mentions = set(entry_data["guild_pings"]) - self.dm_mentions = set(entry_data["dm_pings"]) + + self.guild_pings = set(entry_data["guild_pings"]) if entry_data["guild_pings"] else set() + self.dm_pings = set(entry_data["dm_pings"]) if entry_data["dm_pings"] else set() async def action(self, ctx: FilterContext) -> None: """Add the stored pings to the alert message content.""" - mentions = self.guild_mentions if ctx.channel.guild else self.dm_mentions + mentions = self.guild_pings if ctx.channel.guild else self.dm_pings new_content = " ".join([self._resolve_mention(mention, ctx.channel.guild) for mention in mentions]) ctx.alert_content = f"{new_content} {ctx.alert_content}" @@ -39,8 +40,8 @@ def __or__(self, other: ActionEntry): return NotImplemented return Ping({ - "ping_type": self.guild_mentions | other.guild_mentions, - "dm_ping_type": self.dm_mentions | other.dm_mentions + "ping_type": self.guild_pings | other.guild_pings, + "dm_ping_type": self.dm_pings | other.dm_pings }) @staticmethod diff --git a/bot/exts/filtering/_settings_types/settings_entry.py b/bot/exts/filtering/_settings_types/settings_entry.py index c3a1a8a070..2883deed82 100644 --- a/bot/exts/filtering/_settings_types/settings_entry.py +++ b/bot/exts/filtering/_settings_types/settings_entry.py @@ -46,7 +46,7 @@ def copy(self) -> SettingsEntry: return self.__class__(self.to_dict()) @classmethod - def create(cls, entry_data: Optional[dict[str, Any]]) -> Optional[SettingsEntry]: + def create(cls, entry_data: Optional[dict[str, Any]], *, keep_empty: bool = False) -> Optional[SettingsEntry]: """ Returns a SettingsEntry object from `entry_data` if it holds any value, None otherwise. @@ -55,7 +55,7 @@ def create(cls, entry_data: Optional[dict[str, Any]]) -> Optional[SettingsEntry] """ if entry_data is None: return None - if hasattr(entry_data, "values") and not any(value for value in entry_data.values()): + if not keep_empty and hasattr(entry_data, "values") and not any(value for value in entry_data.values()): return None return cls(entry_data) diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index d092621938..14c6bd13b0 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -4,7 +4,7 @@ import pkgutil from abc import ABC, abstractmethod from collections import defaultdict -from typing import Set +from typing import Any, Iterable, Union import regex @@ -13,7 +13,7 @@ ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIATION_SELECTORS}]]", regex.V1) -def subclasses_in_package(package: str, prefix: str, parent: type) -> Set[type]: +def subclasses_in_package(package: str, prefix: str, parent: type) -> set[type]: """Return all the subclasses of class `parent`, found in the top-level of `package`, given by absolute path.""" subclasses = set() @@ -50,6 +50,22 @@ def past_tense(word: str) -> str: return word + "ed" +def to_serializable(item: Any) -> Union[bool, int, float, str, list, dict, None]: + """Convert the item into an object that can be converted to JSON.""" + if isinstance(item, (bool, int, float, str, type(None))): + return item + if isinstance(item, dict): + result = {} + for key, value in item.items(): + if not isinstance(key, (bool, int, float, str, type(None))): + key = str(key) + result[key] = to_serializable(value) + return result + if isinstance(item, Iterable): + return [to_serializable(subitem) for subitem in item] + return str(item) + + class FieldRequiring(ABC): """A mixin class that can force its concrete subclasses to set a value for specific class attributes.""" diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 2cfb456567..2a24769d06 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -13,15 +13,21 @@ from bot.constants import Colours, MODERATION_ROLES, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter +from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._ui import ArgumentCompletionView -from bot.exts.filtering._utils import past_tense +from bot.exts.filtering._utils import past_tense, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.messages import format_channel, format_user log = get_logger(__name__) +# Max number of characters in a Discord embed field value, minus 6 characters for a placeholder. +MAX_FIELD_SIZE = 1018 +# Max number of characters for an embed field's value before it should take its own line. +MAX_INLINE_SIZE = 50 + class Filtering(Cog): """Filtering and alerting for content posted on the server.""" @@ -161,13 +167,11 @@ async def blocklist(self, ctx: Context) -> None: @blocklist.command(name="list", aliases=("get",)) async def bl_list(self, ctx: Context, list_name: Optional[str] = None) -> None: """List the contents of a specified blacklist.""" - if list_name is None: - await ctx.send( - "The **list_name** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, [], "list_name", list(self.filter_lists), 1, None) - ) + result = self._resolve_list_type_and_name(ctx, ListType.DENY, list_name) + if not result: return - await self._send_list(ctx, list_name, ListType.DENY) + list_type, filter_list = result + await self._send_list(ctx, filter_list, list_type) # endregion # region: whitelist commands @@ -181,48 +185,83 @@ async def allowlist(self, ctx: Context) -> None: @allowlist.command(name="list", aliases=("get",)) async def al_list(self, ctx: Context, list_name: Optional[str] = None) -> None: """List the contents of a specified whitelist.""" - if list_name is None: - await ctx.send( - "The **list_name** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, [], "list_name", list(self.filter_lists), 1, None) - ) + result = self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name) + if not result: return - await self._send_list(ctx, list_name, ListType.ALLOW) + list_type, filter_list = result + await self._send_list(ctx, filter_list, list_type) # endregion # region: filter commands - @commands.group(aliases=("filters", "f")) - async def filter(self, ctx: Context) -> None: - """Group for managing filters.""" - if not ctx.invoked_subcommand: + @commands.group(aliases=("filters", "f"), invoke_without_command=True) + async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: + """ + Group for managing filters. + + If a valid filter ID is provided, an embed describing the filter will be posted. + """ + if not ctx.invoked_subcommand and not id_: await ctx.send_help(ctx.command) + return + + result = self._get_filter_by_id(id_) + if result is None: + await ctx.send(f":x: Could not find a filter with ID `{id_}`.") + return + filter_, filter_list, list_type = result + + # Get filter list settings + default_setting_values = {} + for type_ in ("actions", "validations"): + for _, setting in filter_list.defaults[list_type][type_].items(): + default_setting_values.update(to_serializable(setting.to_dict())) + + # Get the filter's overridden settings + overrides_values = {} + for settings in (filter_.actions, filter_.validations): + if settings: + for _, setting in settings.items(): + overrides_values.update(to_serializable(setting.to_dict())) + + # Combine them. It's done in this way to preserve field order, since the filter won't have all settings. + total_values = {} + for name, value in default_setting_values.items(): + if name not in overrides_values: + total_values[name] = value + else: + total_values[f"{name}*"] = overrides_values[name] + # Add the filter-specific settings. + if hasattr(filter_.extra_fields, "dict"): + extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) + for name, value in filter_.extra_fields.dict().items(): + if name not in extra_fields_overrides: + total_values[f"{filter_.name}/{name}"] = value + else: + total_values[f"{filter_.name}/{name}*"] = value + + embed = self._build_embed_from_dict(total_values) + embed.description = f"`{filter_.content}`" + if filter_.description: + embed.description += f" - {filter_.description}" + embed.set_author(name=f"Filter #{id_} - " + f"{past_tense(list_type.name.lower())} {filter_list.name}".title()) + embed.set_footer(text=( + "Field names with an asterisk have values which override the defaults of the containing filter list. " + f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + )) + await ctx.send(embed=embed) @filter.command(name="list", aliases=("get",)) async def f_list( self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None ) -> None: """List the contents of a specified list of filters.""" - if list_name is None: - await ctx.send( - "The **list_name** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, [list_type], "list_name", list(self.filter_lists), 1, None) - ) + result = await self._resolve_list_type_and_name(ctx, list_type, list_name) + if result is None: return + list_type, filter_list = result - if list_type is None: - filter_list = self._get_list_by_name(list_name) - if len(filter_list.filter_lists) > 1: - await ctx.send( - "The **list_type** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView( - ctx, [list_name], "list_type", [option.name for option in ListType], 0, list_type_converter - ) - ) - return - list_type = list(filter_list.filter_lists)[0] - - await self._send_list(ctx, list_name, list_type) + await self._send_list(ctx, filter_list, list_type) @filter.command(name="describe", aliases=("explain", "manual")) async def f_describe(self, ctx: Context, filter_name: Optional[str]) -> None: @@ -286,22 +325,34 @@ async def filterlist(self, ctx: Context) -> None: if not ctx.invoked_subcommand: await ctx.send_help(ctx.command) - @filterlist.command(name="describe", aliases=("explain", "manual")) - async def fl_describe(self, ctx: Context, filterlist_name: Optional[str]) -> None: - """Show a description of the specified filter list, or a list of possible values if no name is specified.""" - if not filterlist_name: - embed = Embed(description="\n".join(self.filter_lists)) + @filterlist.command(name="describe", aliases=("explain", "manual", "id")) + async def fl_describe( + self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None + ) -> None: + """Show a description of the specified filter list, or a list of possible values if no values are provided.""" + if not list_type and not list_name: + embed = Embed(description="\n".join(f"\u2003 {fl}" for fl in self.filter_lists), colour=Colour.blue()) embed.set_author(name="List of filter lists names") - else: - try: - filter_list = self._get_list_by_name(filterlist_name) - except BadArgument as e: - await ctx.send(f":x: {e}") - return - # Use the class's docstring, and ignore single newlines. - embed = Embed(description=re.sub(r"(? Optional[tuple[ListType, FilterList]]: + """Prompt the user to complete the list type or list name if one of them is missing.""" + if list_name is None: + await ctx.send( + "The **list_name** argument is unspecified. Please pick a value from the options below:", + view=ArgumentCompletionView(ctx, [list_type], "list_name", list(self.filter_lists), 1, None) + ) + return None + + filter_list = self._get_list_by_name(list_name) + if list_type is None: + if len(filter_list.filter_lists) > 1: + await ctx.send( + "The **list_type** argument is unspecified. Please pick a value from the options below:", + view=ArgumentCompletionView( + ctx, [list_name], "list_type", [option.name for option in ListType], 0, list_type_converter + ) + ) + return None + list_type = list(filter_list.filter_lists)[0] + return list_type, filter_list + def _get_list_by_name(self, list_name: str) -> FilterList: """Get a filter list by its name, or raise an error if there's no such list.""" log.trace(f"Getting the filter list matching the name {list_name}") @@ -375,9 +450,9 @@ def _get_list_by_name(self, list_name: str) -> FilterList: log.trace(f"Found list named {filter_list.name}") return filter_list - async def _send_list(self, ctx: Context, list_name: str, list_type: ListType) -> None: + @staticmethod + async def _send_list(ctx: Context, filter_list: FilterList, list_type: ListType) -> None: """Show the list of filters identified by the list name and type.""" - filter_list = self._get_list_by_name(list_name) type_filters = filter_list.filter_lists.get(list_type) if type_filters is None: await ctx.send(f":x: There is no list of {past_tense(list_type.name.lower())} {filter_list.name}s.") @@ -391,6 +466,26 @@ async def _send_list(self, ctx: Context, list_name: str, list_type: ListType) -> await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) + def _get_filter_by_id(self, id_: int) -> Optional[tuple[Filter, FilterList, ListType]]: + """Get the filter object corresponding to the provided ID, along with its containing list and list type.""" + for filter_list in self.filter_lists.values(): + for list_type, sublist in filter_list.filter_lists.items(): + if id_ in sublist: + return sublist[id_], filter_list, list_type + + @staticmethod + def _build_embed_from_dict(data: dict) -> Embed: + """Build a Discord embed by populating fields from the given dict.""" + embed = Embed(description="", colour=Colour.blue()) + for setting, value in data.items(): + if setting.startswith("_"): + continue + value = str(value) if value not in ("", None) else "-" + if len(value) > MAX_FIELD_SIZE: + value = value[:MAX_FIELD_SIZE] + " [...]" + embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) + return embed + # endregion From a13417c82c73a5d30a66ae52eeb280622747ac3a Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 10 Sep 2022 20:31:59 +0300 Subject: [PATCH 012/132] Convert all setting entries to pydnatic models In order to facilitate this change, the init of all setting entries was removed, and as such the base SettingsEntry class doesn't count as abstract anymore, which broke the MUST_SET behavior. It was changed to not raise errors for MUST_SET values of attributes which were set in the current class. Additionally fixed a small bug where filters weren't listed properly in the list command. Despite the pydantic manual not writing it that way, I first made the validators class methods, otherwise it gave linting errors which couldn't be ignored with noqa (because then it complained about blanket noqas). --- .../filtering/_settings_types/bypass_roles.py | 26 +++---- .../_settings_types/channel_scope.py | 46 ++++++------- .../_settings_types/delete_messages.py | 14 ++-- bot/exts/filtering/_settings_types/enabled.py | 12 ++-- .../filtering/_settings_types/filter_dm.py | 10 ++- .../infraction_and_notification.py | 68 ++++++++----------- bot/exts/filtering/_settings_types/ping.py | 25 ++++--- .../filtering/_settings_types/send_alert.py | 12 ++-- .../_settings_types/settings_entry.py | 38 +++-------- bot/exts/filtering/_utils.py | 16 +++-- bot/exts/filtering/filtering.py | 8 +-- 11 files changed, 125 insertions(+), 150 deletions(-) diff --git a/bot/exts/filtering/_settings_types/bypass_roles.py b/bot/exts/filtering/_settings_types/bypass_roles.py index e183e0b425..a5c18cffc9 100644 --- a/bot/exts/filtering/_settings_types/bypass_roles.py +++ b/bot/exts/filtering/_settings_types/bypass_roles.py @@ -1,6 +1,7 @@ -from typing import Any +from typing import ClassVar, Union from discord import Member +from pydantic import validator from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ValidationEntry @@ -9,17 +10,18 @@ class RoleBypass(ValidationEntry): """A setting entry which tells whether the roles the member has allow them to bypass the filter.""" - name = "bypass_roles" - description = "A list of role IDs or role names. Users with these roles will not trigger the filter." - - def __init__(self, entry_data: Any): - super().__init__(entry_data) - self.bypass_roles = set() - for role in entry_data: - if role.isdigit(): - self.bypass_roles.add(int(role)) - else: - self.bypass_roles.add(role) + name: ClassVar[str] = "bypass_roles" + description: ClassVar[str] = "A list of role IDs or role names. Users with these roles will not trigger the filter." + + bypass_roles: set[Union[int, str]] + + @validator("bypass_roles", each_item=True) + @classmethod + def maybe_cast_to_int(cls, role: str) -> Union[int, str]: + """If the string is alphanumeric, cast it to int.""" + if role.isdigit(): + return int(role) + return role def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered on this user given their roles.""" diff --git a/bot/exts/filtering/_settings_types/channel_scope.py b/bot/exts/filtering/_settings_types/channel_scope.py index 3a95834b33..fd5206b814 100644 --- a/bot/exts/filtering/_settings_types/channel_scope.py +++ b/bot/exts/filtering/_settings_types/channel_scope.py @@ -1,21 +1,16 @@ -from typing import Any, Union +from typing import ClassVar, Union + +from pydantic import validator from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ValidationEntry -def maybe_cast_to_int(item: str) -> Union[str, int]: - """Cast the item to int if it consists of only digit, or leave as is otherwise.""" - if item.isdigit(): - return int(item) - return item - - class ChannelScope(ValidationEntry): """A setting entry which tells whether the filter was invoked in a whitelisted channel or category.""" - name = "channel_scope" - description = { + name: ClassVar[str] = "channel_scope" + description: ClassVar[str] = { "disabled_channels": "A list of channel IDs or channel names. The filter will not trigger in these channels.", "disabled_categories": ( "A list of category IDs or category names. The filter will not trigger in these categories." @@ -26,22 +21,25 @@ class ChannelScope(ValidationEntry): ) } - def __init__(self, entry_data: Any): - super().__init__(entry_data) - if entry_data["disabled_channels"]: - self.disabled_channels = set(map(maybe_cast_to_int, entry_data["disabled_channels"])) - else: - self.disabled_channels = set() + disabled_channels: set[Union[str, int]] + disabled_categories: set[Union[str, int]] + enabled_channels: set[Union[str, int]] - if entry_data["disabled_categories"]: - self.disabled_categories = set(map(maybe_cast_to_int, entry_data["disabled_categories"])) - else: - self.disabled_categories = set() + @validator("*", pre=True) + @classmethod + def init_if_sequence_none(cls, sequence: list[str]) -> list[str]: + """Initialize an empty sequence if the value is None.""" + if sequence is None: + return [] + return sequence - if entry_data["enabled_channels"]: - self.enabled_channels = set(map(maybe_cast_to_int, entry_data["enabled_channels"])) - else: - self.enabled_channels = set() + @validator("*", each_item=True) + @classmethod + def maybe_cast_items(cls, channel_or_category: str) -> Union[str, int]: + """Cast to int each value in each sequence if it is alphanumeric.""" + if channel_or_category.isdigit(): + return int(channel_or_category) + return channel_or_category def triggers_on(self, ctx: FilterContext) -> bool: """ diff --git a/bot/exts/filtering/_settings_types/delete_messages.py b/bot/exts/filtering/_settings_types/delete_messages.py index 8de58f8046..710cb0ed8f 100644 --- a/bot/exts/filtering/_settings_types/delete_messages.py +++ b/bot/exts/filtering/_settings_types/delete_messages.py @@ -1,5 +1,5 @@ from contextlib import suppress -from typing import Any +from typing import ClassVar from discord.errors import NotFound @@ -10,12 +10,12 @@ class DeleteMessages(ActionEntry): """A setting entry which tells whether to delete the offending message(s).""" - name = "delete_messages" - description = "A boolean field. If True, the filter being triggered will cause the offending message to be deleted." + name: ClassVar[str] = "delete_messages" + description: ClassVar[str] = ( + "A boolean field. If True, the filter being triggered will cause the offending message to be deleted." + ) - def __init__(self, entry_data: Any): - super().__init__(entry_data) - self.delete_messages: bool = entry_data + delete_messages: bool async def action(self, ctx: FilterContext) -> None: """Delete the context message(s).""" @@ -32,4 +32,4 @@ def __or__(self, other: ActionEntry): if not isinstance(other, DeleteMessages): return NotImplemented - return DeleteMessages(self.delete_messages or other.delete_messages) + return DeleteMessages(delete_messages=self.delete_messages or other.delete_messages) diff --git a/bot/exts/filtering/_settings_types/enabled.py b/bot/exts/filtering/_settings_types/enabled.py index 081ae02b0c..3b5e3e4461 100644 --- a/bot/exts/filtering/_settings_types/enabled.py +++ b/bot/exts/filtering/_settings_types/enabled.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import ClassVar from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ValidationEntry @@ -7,12 +7,12 @@ class Enabled(ValidationEntry): """A setting entry which tells whether the filter is enabled.""" - name = "enabled" - description = "A boolean field. Setting it to False allows disabling the filter without deleting it entirely." + name: ClassVar[str] = "enabled" + description: ClassVar[str] = ( + "A boolean field. Setting it to False allows disabling the filter without deleting it entirely." + ) - def __init__(self, entry_data: Any): - super().__init__(entry_data) - self.enabled = entry_data + enabled: bool def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter is enabled.""" diff --git a/bot/exts/filtering/_settings_types/filter_dm.py b/bot/exts/filtering/_settings_types/filter_dm.py index 1405a636f3..93022320f6 100644 --- a/bot/exts/filtering/_settings_types/filter_dm.py +++ b/bot/exts/filtering/_settings_types/filter_dm.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import ClassVar from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ValidationEntry @@ -7,12 +7,10 @@ class FilterDM(ValidationEntry): """A setting entry which tells whether to apply the filter to DMs.""" - name = "filter_dm" - description = "A boolean field. If True, the filter can trigger for messages sent to the bot in DMs." + name: ClassVar[str] = "filter_dm" + description: ClassVar[str] = "A boolean field. If True, the filter can trigger for messages sent to the bot in DMs." - def __init__(self, entry_data: Any): - super().__init__(entry_data) - self.filter_dm = entry_data + filter_dm: bool def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered even if it was triggered in DMs.""" diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index 4fae09f235..9c7d7b8fff 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -1,11 +1,12 @@ from collections import namedtuple from datetime import timedelta from enum import Enum, auto -from typing import Any, Optional +from typing import ClassVar, Optional import arrow from discord import Colour, Embed from discord.errors import Forbidden +from pydantic import validator import bot from bot.constants import Channels, Guild @@ -50,8 +51,8 @@ class InfractionAndNotification(ActionEntry): Since a DM cannot be sent when a user is banned or kicked, these two functions need to be grouped together. """ - name = "infraction_and_notification" - description = { + name: ClassVar[str] = "infraction_and_notification" + description: ClassVar[dict[str, str]] = { "infraction_type": ( "The type of infraction to issue when the filter triggers, or 'NONE'. " "If two infractions are triggered for the same message, " @@ -65,27 +66,18 @@ class InfractionAndNotification(ActionEntry): "dm_embed": "The contents of the embed to be DMed to the offending user." } - def __init__(self, entry_data: Any): - super().__init__(entry_data) + dm_content: str + dm_embed: str + infraction_type: Optional[Infraction] + infraction_reason: Optional[str] + infraction_duration: Optional[float] + superstar: Optional[superstar] = None - if entry_data["infraction_type"]: - self.infraction_type = entry_data["infraction_type"] - if isinstance(self.infraction_type, str): - self.infraction_type = Infraction[self.infraction_type.replace(" ", "_").upper()] - self.infraction_reason = entry_data["infraction_reason"] - if entry_data["infraction_duration"] is not None: - self.infraction_duration = float(entry_data["infraction_duration"]) - else: - self.infraction_duration = None - else: - self.infraction_type = Infraction.NONE - self.infraction_reason = None - self.infraction_duration = 0 - - self.dm_content = entry_data["dm_content"] - self.dm_embed = entry_data["dm_embed"] - - self._superstar = entry_data.get("superstar", None) + @validator("infraction_type", pre=True) + @classmethod + def convert_infraction_name(cls, infr_type: str) -> Infraction: + """Convert the string to an Infraction by name.""" + return Infraction[infr_type.replace(" ", "_").upper()] if infr_type else Infraction.NONE async def action(self, ctx: FilterContext) -> None: """Send the notification to the user, and apply any specified infractions.""" @@ -115,14 +107,14 @@ async def action(self, ctx: FilterContext) -> None: msg_ctx.guild = bot.instance.get_guild(Guild.id) msg_ctx.author = ctx.author msg_ctx.channel = ctx.channel - if self._superstar: + if self.superstar: msg_ctx.command = bot.instance.get_command("superstarify") await msg_ctx.invoke( msg_ctx.command, ctx.author, - arrow.utcnow() + timedelta(seconds=self._superstar.duration) - if self._superstar.duration is not None else None, - reason=self._superstar.reason + arrow.utcnow() + timedelta(seconds=self.superstar.duration) + if self.superstar.duration is not None else None, + reason=self.superstar.reason ) ctx.action_descriptions.append("superstar") @@ -160,31 +152,31 @@ def __or__(self, other: ActionEntry): # Lower number -> higher in the hierarchy if self.infraction_type.value < other.infraction_type.value and other.infraction_type != Infraction.SUPERSTAR: - result = InfractionAndNotification(self.to_dict()) - result._superstar = self._merge_superstars(self._superstar, other._superstar) + result = self.copy() + result.superstar = self._merge_superstars(self.superstar, other.superstar) return result elif self.infraction_type.value > other.infraction_type.value and self.infraction_type != Infraction.SUPERSTAR: - result = InfractionAndNotification(other.to_dict()) - result._superstar = self._merge_superstars(self._superstar, other._superstar) + result = other.copy() + result.superstar = self._merge_superstars(self.superstar, other.superstar) return result if self.infraction_type == other.infraction_type: if self.infraction_duration is None or ( other.infraction_duration is not None and self.infraction_duration > other.infraction_duration ): - result = InfractionAndNotification(self.to_dict()) + result = self.copy() else: - result = InfractionAndNotification(other.to_dict()) - result._superstar = self._merge_superstars(self._superstar, other._superstar) + result = other.copy() + result.superstar = self._merge_superstars(self.superstar, other.superstar) return result # At this stage the infraction types are different, and the lower one is a superstar. if self.infraction_type.value < other.infraction_type.value: - result = InfractionAndNotification(self.to_dict()) - result._superstar = superstar(other.infraction_reason, other.infraction_duration) + result = self.copy() + result.superstar = superstar(other.infraction_reason, other.infraction_duration) else: - result = InfractionAndNotification(other.to_dict()) - result._superstar = superstar(self.infraction_reason, self.infraction_duration) + result = other.copy() + result.superstar = superstar(self.infraction_reason, self.infraction_duration) return result @staticmethod diff --git a/bot/exts/filtering/_settings_types/ping.py b/bot/exts/filtering/_settings_types/ping.py index 1e00676905..8a3403b59a 100644 --- a/bot/exts/filtering/_settings_types/ping.py +++ b/bot/exts/filtering/_settings_types/ping.py @@ -1,7 +1,8 @@ from functools import cache -from typing import Any +from typing import ClassVar from discord import Guild +from pydantic import validator from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry @@ -10,8 +11,8 @@ class Ping(ActionEntry): """A setting entry which adds the appropriate pings to the alert.""" - name = "mentions" - description = { + name: ClassVar[str] = "mentions" + description: ClassVar[dict[str, str]] = { "guild_pings": ( "A list of role IDs/role names/user IDs/user names/here/everyone. " "If a mod-alert is generated for a filter triggered in a public channel, these will be pinged." @@ -22,11 +23,16 @@ class Ping(ActionEntry): ) } - def __init__(self, entry_data: Any): - super().__init__(entry_data) + guild_pings: set[str] + dm_pings: set[str] - self.guild_pings = set(entry_data["guild_pings"]) if entry_data["guild_pings"] else set() - self.dm_pings = set(entry_data["dm_pings"]) if entry_data["dm_pings"] else set() + @validator("*") + @classmethod + def init_sequence_if_none(cls, pings: list[str]) -> list[str]: + """Initialize an empty sequence if the value is None.""" + if pings is None: + return [] + return pings async def action(self, ctx: FilterContext) -> None: """Add the stored pings to the alert message content.""" @@ -39,10 +45,7 @@ def __or__(self, other: ActionEntry): if not isinstance(other, Ping): return NotImplemented - return Ping({ - "ping_type": self.guild_pings | other.guild_pings, - "dm_ping_type": self.dm_pings | other.dm_pings - }) + return Ping(ping_type=self.guild_pings | other.guild_pings, dm_ping_type=self.dm_pings | other.dm_pings) @staticmethod @cache diff --git a/bot/exts/filtering/_settings_types/send_alert.py b/bot/exts/filtering/_settings_types/send_alert.py index 6429b99ac9..04e4007643 100644 --- a/bot/exts/filtering/_settings_types/send_alert.py +++ b/bot/exts/filtering/_settings_types/send_alert.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import ClassVar from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry @@ -7,12 +7,10 @@ class SendAlert(ActionEntry): """A setting entry which tells whether to send an alert message.""" - name = "send_alert" - description = "A boolean field. If all filters triggered set this to False, no mod-alert will be created." + name: ClassVar[str] = "send_alert" + description: ClassVar[str] = "A boolean. If all filters triggered set this to False, no mod-alert will be created." - def __init__(self, entry_data: Any): - super().__init__(entry_data) - self.send_alert: bool = entry_data + send_alert: bool async def action(self, ctx: FilterContext) -> None: """Add the stored pings to the alert message content.""" @@ -23,4 +21,4 @@ def __or__(self, other: ActionEntry): if not isinstance(other, SendAlert): return NotImplemented - return SendAlert(self.send_alert or other.send_alert) + return SendAlert(send_alert=self.send_alert or other.send_alert) diff --git a/bot/exts/filtering/_settings_types/settings_entry.py b/bot/exts/filtering/_settings_types/settings_entry.py index 2883deed82..2b3b030a05 100644 --- a/bot/exts/filtering/_settings_types/settings_entry.py +++ b/bot/exts/filtering/_settings_types/settings_entry.py @@ -1,13 +1,15 @@ from __future__ import annotations from abc import abstractmethod -from typing import Any, Optional +from typing import Any, ClassVar, Optional, Union + +from pydantic import BaseModel from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._utils import FieldRequiring -class SettingsEntry(FieldRequiring): +class SettingsEntry(BaseModel, FieldRequiring): """ A basic entry in the settings field appearing in every filter list and filter. @@ -16,34 +18,10 @@ class SettingsEntry(FieldRequiring): # Each subclass must define a name matching the entry name we're expecting to receive from the database. # Names must be unique across all filter lists. - name = FieldRequiring.MUST_SET_UNIQUE + name: ClassVar[str] = FieldRequiring.MUST_SET_UNIQUE # Each subclass must define a description of what it does. If the data an entry type receives is comprised of # several DB fields, the value should a dictionary of field names and their descriptions. - description = FieldRequiring.MUST_SET - - @abstractmethod - def __init__(self, entry_data: Any): - super().__init__() - self._dict = {} - - def __setattr__(self, key: str, value: Any) -> None: - super().__setattr__(key, value) - if key == "_dict": - return - self._dict[key] = value - - def __eq__(self, other: SettingsEntry) -> bool: - if not isinstance(other, SettingsEntry): - return NotImplemented - return self._dict == other._dict - - def to_dict(self) -> dict[str, Any]: - """Return a dictionary representation of the entry.""" - return self._dict.copy() - - def copy(self) -> SettingsEntry: - """Return a new entry object with the same parameters.""" - return self.__class__(self.to_dict()) + description: ClassVar[Union[str, dict[str, str]]] = FieldRequiring.MUST_SET @classmethod def create(cls, entry_data: Optional[dict[str, Any]], *, keep_empty: bool = False) -> Optional[SettingsEntry]: @@ -58,7 +36,9 @@ def create(cls, entry_data: Optional[dict[str, Any]], *, keep_empty: bool = Fals if not keep_empty and hasattr(entry_data, "values") and not any(value for value in entry_data.values()): return None - return cls(entry_data) + if not isinstance(entry_data, dict): + entry_data = {cls.name: entry_data} + return cls(**entry_data) class ValidationEntry(SettingsEntry): diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index 14c6bd13b0..158f1e7bd2 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -84,14 +84,18 @@ def __init__(self): ... def __init_subclass__(cls, **kwargs): + def inherited(attr: str) -> bool: + """True if `attr` was inherited from a parent class.""" + for parent in cls.__mro__[1:-1]: # The first element is the class itself, last element is object. + if hasattr(parent, attr): # The attribute was inherited. + return True + return False + # If a new attribute with the value MUST_SET_UNIQUE was defined in an abstract class, record it. if inspect.isabstract(cls): for attribute in dir(cls): if getattr(cls, attribute, None) is FieldRequiring.MUST_SET_UNIQUE: - for parent in cls.__mro__[1:-1]: # The first element is the class itself, last element is object. - if hasattr(parent, attribute): # The attribute was inherited. - break - else: + if not inherited(attribute): # A new attribute with the value MUST_SET_UNIQUE. FieldRequiring.__unique_attributes[cls][attribute] = set() return @@ -100,9 +104,9 @@ def __init_subclass__(cls, **kwargs): if attribute.startswith("__") or attribute in ("MUST_SET", "MUST_SET_UNIQUE"): continue value = getattr(cls, attribute) - if value is FieldRequiring.MUST_SET: + if value is FieldRequiring.MUST_SET and inherited(attribute): raise ValueError(f"You must set attribute {attribute!r} when creating {cls!r}") - elif value is FieldRequiring.MUST_SET_UNIQUE: + elif value is FieldRequiring.MUST_SET_UNIQUE and inherited(attribute): raise ValueError(f"You must set a unique value to attribute {attribute!r} when creating {cls!r}") else: # Check if the value needs to be unique. diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 2a24769d06..630474c133 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -215,14 +215,14 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: default_setting_values = {} for type_ in ("actions", "validations"): for _, setting in filter_list.defaults[list_type][type_].items(): - default_setting_values.update(to_serializable(setting.to_dict())) + default_setting_values.update(to_serializable(setting.dict())) # Get the filter's overridden settings overrides_values = {} for settings in (filter_.actions, filter_.validations): if settings: for _, setting in settings.items(): - overrides_values.update(to_serializable(setting.to_dict())) + overrides_values.update(to_serializable(setting.dict())) # Combine them. It's done in this way to preserve field order, since the filter won't have all settings. total_values = {} @@ -345,7 +345,7 @@ async def fl_describe( setting_values = {} for type_ in ("actions", "validations"): for _, setting in list_defaults[type_].items(): - setting_values.update(to_serializable(setting.to_dict())) + setting_values.update(to_serializable(setting.dict())) embed = self._build_embed_from_dict(setting_values) # Use the class's docstring, and ignore single newlines. @@ -458,7 +458,7 @@ async def _send_list(ctx: Context, filter_list: FilterList, list_type: ListType) await ctx.send(f":x: There is no list of {past_tense(list_type.name.lower())} {filter_list.name}s.") return - lines = list(map(str, type_filters)) + lines = list(map(str, type_filters.values())) log.trace(f"Sending a list of {len(lines)} filters.") embed = Embed(colour=Colour.blue()) From f10d3de8699ae52e656396e9fdac0f6f37f2fe47 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 29 Sep 2022 22:11:50 +0300 Subject: [PATCH 013/132] Filter adding commands, simplify infraction This commit adds the mechanic to add new filters. The overrides of the settings can be provided in the command itself, but also through a UI made of Discord components. The UI adjusts itself to the data type of the setting, e.g a boolean setting will invoke a select with "True" and "False" options. This commit additionally gets rid of the mechanic to apply a superstar alongside another higher tier infraction for the same message. This is an edge case that isn't worth the complexity at the moment. Includes a small fix to the Ping setting which made the __or__ method malfunction. --- bot/exts/filtering/_filter_lists/domain.py | 4 + bot/exts/filtering/_filter_lists/extension.py | 4 + .../filtering/_filter_lists/filter_list.py | 24 +- bot/exts/filtering/_filter_lists/invite.py | 4 + bot/exts/filtering/_filter_lists/token.py | 4 + bot/exts/filtering/_filters/filter.py | 16 + .../infraction_and_notification.py | 95 +-- bot/exts/filtering/_settings_types/ping.py | 2 +- bot/exts/filtering/_ui.py | 621 +++++++++++++++++- bot/exts/filtering/filtering.py | 217 ++++-- 10 files changed, 869 insertions(+), 122 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index 7f92b62e87..6e12dad411 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -36,6 +36,10 @@ def __init__(self, filtering_cog: Filtering): super().__init__(DomainFilter) filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + def get_filter_type(self, content: str) -> Type[Filter]: + """Get a subclass of filter matching the filter list and the filter's content.""" + return DomainFilter + @property def filter_types(self) -> set[Type[Filter]]: """Return the types of filters used by this list.""" diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index 2447bebde7..e34ead3935 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -53,6 +53,10 @@ def __init__(self, filtering_cog: Filtering): filtering_cog.subscribe(self, Event.MESSAGE) self._whitelisted_description = None + def get_filter_type(self, content: str) -> Type[Filter]: + """Get a subclass of filter matching the filter list and the filter's content.""" + return ExtensionFilter + @property def filter_types(self) -> set[Type[Filter]]: """Return the types of filters used by this list.""" diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 3b5138fe44..c34f46878b 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -44,6 +44,7 @@ class FilterList(FieldRequiring): name = FieldRequiring.MUST_SET_UNIQUE def __init__(self, filter_type: Type[Filter]): + self.list_ids = {} self.filter_lists: dict[ListType, dict[int, Filter]] = {} self.defaults = {} @@ -54,14 +55,25 @@ def add_list(self, list_data: dict) -> None: actions, validations = create_settings(list_data["settings"], keep_empty=True) list_type = ListType(list_data["list_type"]) self.defaults[list_type] = {"actions": actions, "validations": validations} + self.list_ids[list_type] = list_data["id"] - filters = {} + self.filter_lists[list_type] = {} for filter_data in list_data["filters"]: - try: - filters[filter_data["id"]] = self.filter_type(filter_data) - except TypeError as e: - log.warning(e) - self.filter_lists[list_type] = filters + self.add_filter(filter_data, list_type) + + def add_filter(self, filter_data: dict, list_type: ListType) -> Filter: + """Add a filter to the list of the specified type.""" + try: + new_filter = self.filter_type(filter_data) + self.filter_lists[list_type][filter_data["id"]] = new_filter + except TypeError as e: + log.warning(e) + else: + return new_filter + + @abstractmethod + def get_filter_type(self, content: str) -> Type[Filter]: + """Get a subclass of filter matching the filter list and the filter's content.""" @property @abstractmethod diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 4e8d74d8a3..095699597c 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -42,6 +42,10 @@ def __init__(self, filtering_cog: Filtering): super().__init__(InviteFilter) filtering_cog.subscribe(self, Event.MESSAGE) + def get_filter_type(self, content: str) -> Type[Filter]: + """Get a subclass of filter matching the filter list and the filter's content.""" + return InviteFilter + @property def filter_types(self) -> set[Type[Filter]]: """Return the types of filters used by this list.""" diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index c989b06b9e..aca0fdedf2 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -37,6 +37,10 @@ def __init__(self, filtering_cog: Filtering): super().__init__(TokenFilter) filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + def get_filter_type(self, content: str) -> Type[Filter]: + """Get a subclass of filter matching the filter list and the filter's content.""" + return TokenFilter + @property def filter_types(self) -> set[Type[Filter]]: """Return the types of filters used by this list.""" diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index da149dce60..92393871ad 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,4 +1,7 @@ from abc import abstractmethod +from typing import Optional + +from pydantic import ValidationError from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings import create_settings @@ -32,6 +35,19 @@ def __init__(self, filter_data: dict): def triggered_on(self, ctx: FilterContext) -> bool: """Search for the filter's content within a given context.""" + @classmethod + def validate_filter_settings(cls, extra_fields: dict) -> tuple[bool, Optional[str]]: + """Validate whether the supplied fields are valid for the filter, and provide the error message if not.""" + if cls.extra_fields_type is None: + return True, None + + try: + cls.extra_fields_type(**extra_fields) + except ValidationError as e: + return False, repr(e) + else: + return True, None + def __str__(self) -> str: """A string representation of the filter.""" string = f"#{self.id}. `{self.content}`" diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index 9c7d7b8fff..c1fc007605 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -1,7 +1,6 @@ -from collections import namedtuple from datetime import timedelta from enum import Enum, auto -from typing import ClassVar, Optional +from typing import ClassVar import arrow from discord import Colour, Embed @@ -25,25 +24,11 @@ class Infraction(Enum): WARNING = auto() WATCH = auto() NOTE = auto() - NONE = auto() # Allows making operations on an entry with no infraction without checking for None. - - def __bool__(self) -> bool: - """ - Make the NONE value false-y. - - This is useful for Settings.create to evaluate whether the entry contains anything. - """ - return self != Infraction.NONE def __str__(self) -> str: - if self == Infraction.NONE: - return "" return self.name -superstar = namedtuple("superstar", ["reason", "duration"]) - - class InfractionAndNotification(ActionEntry): """ A setting entry which specifies what infraction to issue and the notification to DM the user. @@ -66,23 +51,22 @@ class InfractionAndNotification(ActionEntry): "dm_embed": "The contents of the embed to be DMed to the offending user." } - dm_content: str - dm_embed: str - infraction_type: Optional[Infraction] - infraction_reason: Optional[str] - infraction_duration: Optional[float] - superstar: Optional[superstar] = None + dm_content: str | None + dm_embed: str | None + infraction_type: Infraction | None + infraction_reason: str | None + infraction_duration: float | None @validator("infraction_type", pre=True) @classmethod def convert_infraction_name(cls, infr_type: str) -> Infraction: """Convert the string to an Infraction by name.""" - return Infraction[infr_type.replace(" ", "_").upper()] if infr_type else Infraction.NONE + return Infraction[infr_type.replace(" ", "_").upper()] if infr_type else None async def action(self, ctx: FilterContext) -> None: """Send the notification to the user, and apply any specified infractions.""" # If there is no infraction to apply, any DM contents already provided in the context take precedence. - if self.infraction_type == Infraction.NONE and (ctx.dm_content or ctx.dm_embed): + if self.infraction_type is None and (ctx.dm_content or ctx.dm_embed): dm_content = ctx.dm_content dm_embed = ctx.dm_embed else: @@ -107,21 +91,11 @@ async def action(self, ctx: FilterContext) -> None: msg_ctx.guild = bot.instance.get_guild(Guild.id) msg_ctx.author = ctx.author msg_ctx.channel = ctx.channel - if self.superstar: - msg_ctx.command = bot.instance.get_command("superstarify") - await msg_ctx.invoke( - msg_ctx.command, - ctx.author, - arrow.utcnow() + timedelta(seconds=self.superstar.duration) - if self.superstar.duration is not None else None, - reason=self.superstar.reason - ) - ctx.action_descriptions.append("superstar") - if self.infraction_type != Infraction.NONE: + if self.infraction_type is not None: if self.infraction_type == Infraction.BAN or not hasattr(ctx.channel, "guild"): msg_ctx.channel = bot.instance.get_channel(Channels.mod_alerts) - msg_ctx.command = bot.instance.get_command(self.infraction_type.name) + msg_ctx.command = bot.instance.get_command(self.infraction_type.name.lower()) await msg_ctx.invoke( msg_ctx.command, ctx.author, @@ -137,12 +111,6 @@ def __or__(self, other: ActionEntry): If the infractions are different, take the data of the one higher up the hierarchy. - A special case is made for superstar infractions. Even if we decide to auto-mute a user, if they have a - particularly problematic username we will still want to superstarify them. - - This is a "best attempt" implementation. Trying to account for any type of combination would create an - extremely complex ruleset. For example, we could special-case watches as well. - There is no clear way to properly combine several notification messages, especially when it's in two parts. To avoid bombarding the user with several notifications, the message with the more significant infraction is used. @@ -151,42 +119,19 @@ def __or__(self, other: ActionEntry): return NotImplemented # Lower number -> higher in the hierarchy - if self.infraction_type.value < other.infraction_type.value and other.infraction_type != Infraction.SUPERSTAR: - result = self.copy() - result.superstar = self._merge_superstars(self.superstar, other.superstar) - return result - elif self.infraction_type.value > other.infraction_type.value and self.infraction_type != Infraction.SUPERSTAR: - result = other.copy() - result.superstar = self._merge_superstars(self.superstar, other.superstar) - return result - - if self.infraction_type == other.infraction_type: + if self.infraction_type is None: + return other.copy() + elif other.infraction_type is None: + return self.copy() + elif self.infraction_type.value < other.infraction_type.value: + return self.copy() + elif self.infraction_type.value > other.infraction_type.value: + return other.copy() + else: if self.infraction_duration is None or ( - other.infraction_duration is not None and self.infraction_duration > other.infraction_duration + other.infraction_duration is not None and self.infraction_duration > other.infraction_duration ): result = self.copy() else: result = other.copy() - result.superstar = self._merge_superstars(self.superstar, other.superstar) return result - - # At this stage the infraction types are different, and the lower one is a superstar. - if self.infraction_type.value < other.infraction_type.value: - result = self.copy() - result.superstar = superstar(other.infraction_reason, other.infraction_duration) - else: - result = other.copy() - result.superstar = superstar(self.infraction_reason, self.infraction_duration) - return result - - @staticmethod - def _merge_superstars(superstar1: Optional[superstar], superstar2: Optional[superstar]) -> Optional[superstar]: - """Take the superstar with the greater duration.""" - if not superstar1: - return superstar2 - if not superstar2: - return superstar1 - - if superstar1.duration is None or superstar1.duration > superstar2.duration: - return superstar1 - return superstar2 diff --git a/bot/exts/filtering/_settings_types/ping.py b/bot/exts/filtering/_settings_types/ping.py index 8a3403b59a..0bfc12809b 100644 --- a/bot/exts/filtering/_settings_types/ping.py +++ b/bot/exts/filtering/_settings_types/ping.py @@ -45,7 +45,7 @@ def __or__(self, other: ActionEntry): if not isinstance(other, Ping): return NotImplemented - return Ping(ping_type=self.guild_pings | other.guild_pings, dm_ping_type=self.dm_pings | other.dm_pings) + return Ping(guild_pings=self.guild_pings | other.guild_pings, dm_pings=self.dm_pings | other.dm_pings) @staticmethod @cache diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index efedb2c0c8..d40d4d41e7 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -1,13 +1,42 @@ -from typing import Callable, Optional +from __future__ import annotations + +import re +from enum import EnumMeta +from functools import partial +from typing import Any, Callable, Coroutine, Optional, TypeVar, Union import discord import discord.ui -from discord.ext.commands import Context +from botcore.site_api import ResponseCodeError +from botcore.utils import scheduling +from discord import Embed, Interaction, User +from discord.ext.commands import BadArgument, Context +from discord.ui.select import MISSING, SelectOption +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._utils import to_serializable from bot.log import get_logger log = get_logger(__name__) +# Max number of characters in a Discord embed field value, minus 6 characters for a placeholder. +MAX_FIELD_SIZE = 1018 +# Max number of characters for an embed field's value before it should take its own line. +MAX_INLINE_SIZE = 50 +# Number of seconds before a settings editing view timeout. +EDIT_TIMEOUT = 600 +# Number of seconds before timeout of an editing component. +COMPONENT_TIMEOUT = 180 +# Max length of modal title +MAX_MODAL_TITLE_LENGTH = 45 +# Max length of modal text component label +MAX_MODAL_LABEL_LENGTH = 45 +# Max number of items in a select +MAX_SELECT_ITEMS = 25 + +T = TypeVar('T') + class ArgumentCompletionSelect(discord.ui.Select): """A select detailing the options that can be picked to assign to a missing argument.""" @@ -66,3 +95,591 @@ async def interaction_check(self, interaction: discord.Interaction) -> bool: await interaction.response.send_message(embed=embed, ephemeral=True) return False return True + + +def build_filter_repr_dict( + filter_list: FilterList, + list_type: ListType, + filter_type: type[Filter], + settings_overrides: dict, + extra_fields_overrides: dict +) -> dict: + """Build a dictionary of field names and values to pass to `_build_embed_from_dict`.""" + # Get filter list settings + default_setting_values = {} + for type_ in ("actions", "validations"): + for _, setting in filter_list.defaults[list_type][type_].items(): + default_setting_values.update(to_serializable(setting.dict())) + + # Add overrides. It's done in this way to preserve field order, since the filter won't have all settings. + total_values = {} + for name, value in default_setting_values.items(): + if name not in settings_overrides: + total_values[name] = value + else: + total_values[f"{name}*"] = settings_overrides[name] + + # Add the filter-specific settings. + if filter_type.extra_fields_type: + # This iterates over the default values of the extra fields model. + for name, value in filter_type.extra_fields_type().dict().items(): + if name not in extra_fields_overrides: + total_values[f"{filter_type.name}/{name}"] = value + else: + total_values[f"{filter_type.name}/{name}*"] = value + + return total_values + + +def populate_embed_from_dict(embed: Embed, data: dict) -> None: + """Populate a Discord embed by populating fields from the given dict.""" + for setting, value in data.items(): + if setting.startswith("_"): + continue + if type(value) in (set, tuple): + value = list(value) + value = str(value) if value not in ("", None) else "-" + if len(value) > MAX_FIELD_SIZE: + value = value[:MAX_FIELD_SIZE] + " [...]" + embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) + + +class CustomCallbackSelect(discord.ui.Select): + """A selection which calls the provided callback on interaction.""" + + def __init__( + self, + callback: Callable[[Interaction, discord.ui.Select], Coroutine[None]], + *, + custom_id: str = MISSING, + placeholder: str | None = None, + min_values: int = 1, + max_values: int = 1, + options: list[SelectOption] = MISSING, + disabled: bool = False, + row: int | None = None, + ): + super().__init__( + custom_id=custom_id, + placeholder=placeholder, + min_values=min_values, + max_values=max_values, + options=options, + disabled=disabled, + row=row + ) + self.custom_callback = callback + + async def callback(self, interaction: Interaction) -> Any: + """Invoke the provided callback.""" + await self.custom_callback(interaction, self) + + +class EditContentModal(discord.ui.Modal, title="Edit Content"): + """A modal to input a filter's content.""" + + content = discord.ui.TextInput(label="Content") + + def __init__(self, embed_view: SettingsEditView, message: discord.Message): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.embed_view = embed_view + self.message = message + + async def on_submit(self, interaction: Interaction) -> None: + """Update the embed with the new content.""" + await interaction.response.defer() + await self.embed_view.update_embed(self.message, content=self.content.value) + + +class EditDescriptionModal(discord.ui.Modal, title="Edit Description"): + """A modal to input a filter's description.""" + + description = discord.ui.TextInput(label="Description") + + def __init__(self, embed_view: SettingsEditView, message: discord.Message): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.embed_view = embed_view + self.message = message + + async def on_submit(self, interaction: Interaction) -> None: + """Update the embed with the new description.""" + await interaction.response.defer() + await self.embed_view.update_embed(self.message, description=self.description.value) + + +class BooleanSelectView(discord.ui.View): + """A view containing an instance of BooleanSelect.""" + + class BooleanSelect(discord.ui.Select): + """Select a true or false value and send it to the supplied callback.""" + + def __init__(self, setting_name: str, update_callback: Callable): + super().__init__(options=[SelectOption(label="True"), SelectOption(label="False")]) + self.setting_name = setting_name + self.update_callback = update_callback + + async def callback(self, interaction: Interaction) -> Any: + """Respond to the interaction by sending the boolean value to the update callback.""" + await interaction.response.edit_message(content="✅ Edit confirmed", view=None) + value = self.values[0] == "True" + await self.update_callback(setting_name=self.setting_name, setting_value=value) + + def __init__(self, setting_name: str, update_callback: Callable): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.add_item(self.BooleanSelect(setting_name, update_callback)) + + +class FreeInputModal(discord.ui.Modal): + """A modal to freely enter a value for a setting.""" + + def __init__(self, setting_name: str, required: bool, type_: type, update_callback: Callable): + title = f"{setting_name} Input" if len(setting_name) < MAX_MODAL_TITLE_LENGTH - 6 else "Setting Input" + super().__init__(timeout=COMPONENT_TIMEOUT, title=title) + + self.setting_name = setting_name + self.type_ = type_ + self.update_callback = update_callback + + label = setting_name if len(setting_name) < MAX_MODAL_TITLE_LENGTH else "Value" + self.setting_input = discord.ui.TextInput(label=label, style=discord.TextStyle.paragraph, required=required) + self.add_item(self.setting_input) + + async def on_submit(self, interaction: Interaction) -> None: + """Update the setting with the new value in the embed.""" + try: + value = self.type_(self.setting_input.value) or None + except (ValueError, TypeError): + await interaction.response.send_message( + f"Could not process the input value for `{self.setting_name}`.", ephemeral=True + ) + else: + await interaction.response.defer() + await self.update_callback(setting_name=self.setting_name, setting_value=value) + + +class SequenceEditView(discord.ui.View): + """A view to modify the contents of a sequence of values.""" + + class SingleItemModal(discord.ui.Modal): + """A modal to enter a single list item.""" + + new_item = discord.ui.TextInput(label="New Item") + + def __init__(self, view: SequenceEditView): + super().__init__(title="Item Addition", timeout=COMPONENT_TIMEOUT) + self.view = view + + async def on_submit(self, interaction: Interaction) -> None: + """Send the submitted value to be added to the list.""" + await self.view.apply_addition(interaction, self.new_item.value) + + class NewListModal(discord.ui.Modal): + """A modal to enter new contents for the list.""" + + new_value = discord.ui.TextInput(label="Enter comma separated values", style=discord.TextStyle.paragraph) + + def __init__(self, view: SequenceEditView): + super().__init__(title="New List", timeout=COMPONENT_TIMEOUT) + self.view = view + + async def on_submit(self, interaction: Interaction) -> None: + """Send the submitted value to be added to the list.""" + await self.view.apply_edit(interaction, self.new_value.value) + + def __init__(self, setting_name: str, starting_value: list, type_: type, update_callback: Callable): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.setting_name = setting_name + self.stored_value = starting_value + self.type_ = type_ + self.update_callback = update_callback + + options = [SelectOption(label=item) for item in starting_value[:MAX_SELECT_ITEMS]] + self.removal_select = CustomCallbackSelect( + self.apply_removal, placeholder="Enter an item to remove", options=options, row=1 + ) + if starting_value: + self.add_item(self.removal_select) + + async def apply_removal(self, interaction: Interaction, select: discord.ui.Select) -> None: + """Remove an item from the list.""" + self.stored_value.remove(select.values[0]) + select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] + if not self.stored_value: + self.remove_item(self.removal_select) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + + async def apply_addition(self, interaction: Interaction, item: str) -> None: + """Add an item to the list.""" + self.stored_value.append(item) + self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] + if len(self.stored_value) == 1: + self.add_item(self.removal_select) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + + async def apply_edit(self, interaction: Interaction, new_list: str) -> None: + """Change the contents of the list.""" + self.stored_value = new_list.split(",") + self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] + if len(self.stored_value) == 1: + self.add_item(self.removal_select) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + + @discord.ui.button(label="Add Value") + async def add_value(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to add an item to the list.""" + await interaction.response.send_modal(self.SingleItemModal(self)) + + @discord.ui.button(label="Free Input") + async def free_input(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to change the entire list.""" + await interaction.response.send_modal(self.NewListModal(self)) + + @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Send the final value to the embed editor.""" + final_value = self.type_(self.stored_value) + # Edit first, it might time out otherwise. + await interaction.response.edit_message(content="✅ Edit confirmed", view=None) + await self.update_callback(setting_name=self.setting_name, setting_value=final_value) + self.stop() + + @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the list editing.""" + await interaction.response.edit_message(content="🚫 Canceled", view=None) + self.stop() + + +class EnumSelectView(discord.ui.View): + """A view containing an instance of EnumSelect.""" + + class EnumSelect(discord.ui.Select): + """Select an enum value and send it to the supplied callback.""" + + def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): + super().__init__(options=[SelectOption(label=elem.name) for elem in enum_cls]) + self.setting_name = setting_name + self.enum_cls = enum_cls + self.update_callback = update_callback + + async def callback(self, interaction: Interaction) -> Any: + """Respond to the interaction by sending the enum value to the update callback.""" + await interaction.response.edit_message(content="✅ Edit confirmed", view=None) + await self.update_callback(setting_name=self.setting_name, setting_value=self.values[0]) + + def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.add_item(self.EnumSelect(setting_name, enum_cls, update_callback)) + + +class SettingsEditView(discord.ui.View): + """A view used to edit a filter's settings before updating the database.""" + + class _REMOVE: + """Sentinel value for when an override should be removed.""" + + def __init__( + self, + filter_list: FilterList, + list_type: ListType, + filter_type: type[Filter], + content: str | None, + description: str | None, + settings_overrides: dict, + filter_settings_overrides: dict, + loaded_settings: dict, + loaded_filter_settings: dict, + author: User, + embed: Embed, + confirm_callback: Callable + ): + super().__init__(timeout=EDIT_TIMEOUT) + self.filter_list = filter_list + self.list_type = list_type + self.filter_type = filter_type + self.content = content + self.description = description + self.settings_overrides = settings_overrides + self.filter_settings_overrides = filter_settings_overrides + self.loaded_settings = loaded_settings + self.loaded_filter_settings = loaded_filter_settings + self.author = author + self.embed = embed + self.confirm_callback = confirm_callback + + all_settings_repr_dict = build_filter_repr_dict( + filter_list, list_type, filter_type, settings_overrides, filter_settings_overrides + ) + populate_embed_from_dict(embed, all_settings_repr_dict) + + self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} + self.type_per_setting_name.update({ + f"{filter_type.name}/{name}": type_ + for name, (_, _, type_) in loaded_filter_settings.get(filter_type.name, {}).items() + }) + + add_select = CustomCallbackSelect( + self._prompt_new_override, + placeholder="Select a setting to edit", + options=[SelectOption(label=name) for name in self.type_per_setting_name], + row=1 + ) + self.add_item(add_select) + + override_names = list(settings_overrides) + list(filter_settings_overrides) + remove_select = CustomCallbackSelect( + self._remove_override, + placeholder="Select an override to remove", + options=[SelectOption(label=name) for name in override_names], + row=2 + ) + if remove_select.options: + self.add_item(remove_select) + + async def interaction_check(self, interaction: Interaction) -> bool: + """Only allow interactions from the command invoker.""" + return interaction.user.id == self.author.id + + @discord.ui.button(label="Edit Content", row=3) + async def edit_content(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to edit the filter's content. Pressing the button invokes a modal.""" + modal = EditContentModal(self, interaction.message) + await interaction.response.send_modal(modal) + + @discord.ui.button(label="Edit Description", row=3) + async def edit_description(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to edit the filter's description. Pressing the button invokes a modal.""" + modal = EditDescriptionModal(self, interaction.message) + await interaction.response.send_modal(modal) + + @discord.ui.button(label="Empty Description", row=3) + async def empty_description(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to empty the filter's description.""" + await self.update_embed(interaction, description=self._REMOVE) + + @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green, row=4) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Confirm the content, description, and settings, and update the filters database.""" + if self.content is None: + await interaction.response.send_message( + ":x: Cannot add a filter with no content.", ephemeral=True, reference=interaction.message + ) + if self.description is None: + self.description = "" + await interaction.response.edit_message(view=None) # Make sure the interaction succeeds first. + try: + await self.confirm_callback( + interaction.message, + self.filter_list, + self.list_type, + self.filter_type, + self.content, + self.description, + self.settings_overrides, + self.filter_settings_overrides + ) + except ResponseCodeError as e: + await interaction.message.channel.send(f"An error occurred: ```{e}```", reference=interaction.message) + + self.stop() + + @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red, row=4) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the operation.""" + await interaction.response.edit_message(content="🚫 Operation canceled.", embed=None, view=None) + self.stop() + + async def _prompt_new_override(self, interaction: Interaction, select: discord.ui.Select) -> None: + """Prompt the user to give an override value for the setting they selected, and respond to the interaction.""" + setting_name = select.values[0] + type_ = self.type_per_setting_name[setting_name] + is_optional, type_ = _remove_optional(type_) + if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias + type_ = type_.__origin__ + new_view = self.copy() + # This is in order to not block the interaction response. There's a potential race condition here, since + # a view's method is used without guaranteeing the task completed, but since it depends on user input + # realistically it shouldn't happen. + scheduling.create_task(interaction.message.edit(view=new_view)) + update_callback = partial(new_view.update_embed, interaction_or_msg=interaction.message) + if type_ is bool: + view = BooleanSelectView(setting_name, update_callback) + await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) + elif type_ in (set, list, tuple): + current_value = self.settings_overrides.get(setting_name, []) + await interaction.response.send_message( + f"Current list: {current_value}", + view=SequenceEditView(setting_name, current_value, type_, update_callback), + ephemeral=True + ) + elif isinstance(type_, EnumMeta): + view = EnumSelectView(setting_name, type_, update_callback) + await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) + else: + await interaction.response.send_modal(FreeInputModal(setting_name, not is_optional, type_, update_callback)) + self.stop() + + async def update_embed( + self, + interaction_or_msg: discord.Interaction | discord.Message, + *, + content: str | None = None, + description: str | type[SettingsEditView._REMOVE] | None = None, + setting_name: str | None = None, + setting_value: str | type[SettingsEditView._REMOVE] | None = None, + ) -> None: + """ + Update the embed with the new information. + + If a setting name is provided with a _REMOVE value, remove the override. + If `interaction_or_msg` is a Message, the invoking Interaction must be deferred before calling this function. + """ + if content is not None or description is not None: + if content is not None: + self.content = content + else: + content = self.content # If there's no content or description, use the existing values. + if description is self._REMOVE: + self.description = None + elif description is not None: + self.description = description + else: + description = self.description + + # Update the embed with the new content and/or description. + self.embed.description = f"`{content}`" if content else "*No content*" + if description is not None and description is not self._REMOVE: + self.embed.description += f" - {description}" + + if setting_name: + # Find the right dictionary to update. + if "/" in setting_name: + filter_name, setting_name = setting_name.split("/", maxsplit=1) + dict_to_edit = self.filter_settings_overrides[filter_name] + else: + dict_to_edit = self.settings_overrides + # Update the setting override value or remove it + if setting_value is not self._REMOVE: + dict_to_edit[setting_name] = setting_value + else: + del dict_to_edit[setting_name] + + # This is inefficient, but otherwise the selects go insane if the user attempts to edit the same setting + # multiple times, even when replacing the select with a new one. + self.embed.clear_fields() + new_view = self.copy() + + if isinstance(interaction_or_msg, discord.Interaction): + await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) + else: + await interaction_or_msg.edit(embed=self.embed, view=new_view) + self.stop() + + async def edit_setting_override(self, interaction: Interaction, setting_name: str, override_value: Any) -> None: + """ + Update the overrides with the new value and edit the embed. + + The interaction needs to be the selection of the setting attached to the embed. + """ + await self.update_embed(interaction, setting_name=setting_name, setting_value=override_value) + + async def _remove_override(self, interaction: Interaction, select: discord.ui.Select) -> None: + """ + Remove the override for the setting the user selected, and edit the embed. + + The interaction needs to be the selection of the setting attached to the embed. + """ + await self.update_embed(interaction, setting_name=select.values[0], setting_value=self._REMOVE) + + def copy(self) -> SettingsEditView: + """Create a copy of this view.""" + return SettingsEditView( + self.filter_list, + self.list_type, + self.filter_type, + self.content, + self.description, + self.settings_overrides, + self.filter_settings_overrides, + self.loaded_settings, + self.loaded_filter_settings, + self.author, + self.embed, + self.confirm_callback + ) + + +def _remove_optional(type_: type) -> tuple[bool, type]: + """Return whether the type is Optional, and the Union of types which aren't None.""" + if not hasattr(type_, "__args__"): + return False, type_ + args = list(type_.__args__) + if type(None) not in args: + return False, type_ + args.remove(type(None)) + return True, Union[tuple(args)] + + +def _parse_value(value: str, type_: type[T]) -> T: + """Parse the value and attempt to convert it to the provided type.""" + is_optional, type_ = _remove_optional(type_) + if is_optional and value == '""': + return None + if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias + type_ = type_.__origin__ + if type_ in (tuple, list, set): + return type_(value.split(",")) + if type_ is bool: + return value == "True" + if isinstance(type_, EnumMeta): + return type_[value.upper()] + + return type_(value) + + +def description_and_settings_converter( + list_name: str, loaded_settings: dict, loaded_filter_settings: dict, input_data: str +) -> tuple[str, dict[str, Any], dict[str, Any]]: + """Parse a string representing a possible description and setting overrides, and validate the setting names.""" + if not input_data: + return "", {}, {} + + settings_pattern = re.compile(r"\s+(?=\S+=\S+)") + single_setting_pattern = re.compile(r"\w+=.+") + + parsed = settings_pattern.split(input_data) + if not parsed: + return "", {}, {} + + description = "" + if not single_setting_pattern.match(parsed[0]): + description, *parsed = parsed + + settings = {setting: value for setting, value in [part.split("=", maxsplit=1) for part in parsed]} + + filter_settings = {} + for setting, _ in list(settings.items()): + if setting not in loaded_settings: + if "/" in setting: + setting_list_name, filter_setting_name = setting.split("/", maxsplit=1) + if setting_list_name.lower() != list_name.lower(): + raise BadArgument( + f"A setting for a {setting_list_name!r} filter was provided, but the list name is {list_name!r}" + ) + if filter_setting_name not in loaded_filter_settings[list_name]: + raise BadArgument(f"{setting!r} is not a recognized setting.") + type_ = loaded_filter_settings[list_name][filter_setting_name][2] + try: + filter_settings[filter_setting_name] = _parse_value(settings.pop(setting), type_) + except (TypeError, ValueError) as e: + raise BadArgument(e) + else: + raise BadArgument(f"{setting!r} is not a recognized setting.") + else: + type_ = loaded_settings[setting][2] + try: + settings[setting] = _parse_value(settings.pop(setting), type_) + except (TypeError, ValueError) as e: + raise BadArgument(e) + + return description, settings, filter_settings diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 630474c133..7ffa121e13 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1,21 +1,26 @@ +import json import operator import re from collections import defaultdict from functools import reduce -from typing import Optional +from typing import Literal, Optional, get_type_hints from discord import Colour, Embed, HTTPException, Message from discord.ext import commands from discord.ext.commands import BadArgument, Cog, Context, has_any_role from discord.utils import escape_markdown +import bot +import bot.exts.filtering._ui as filters_ui from bot.bot import Bot from bot.constants import Colours, MODERATION_ROLES, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings -from bot.exts.filtering._ui import ArgumentCompletionView +from bot.exts.filtering._ui import ( + ArgumentCompletionView, build_filter_repr_dict, description_and_settings_converter, populate_embed_from_dict +) from bot.exts.filtering._utils import past_tense, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator @@ -23,11 +28,6 @@ log = get_logger(__name__) -# Max number of characters in a Discord embed field value, minus 6 characters for a placeholder. -MAX_FIELD_SIZE = 1018 -# Max number of characters for an embed field's value before it should take its own line. -MAX_INLINE_SIZE = 50 - class Filtering(Cog): """Filtering and alerting for content posted on the server.""" @@ -111,15 +111,18 @@ def collect_loaded_types(self) -> None: settings_types.update(type(setting) for _, setting in list_defaults["actions"].items()) settings_types.update(type(setting) for _, setting in list_defaults["validations"].items()) for setting_type in settings_types: + type_hints = get_type_hints(setting_type) # The description should be either a string or a dictionary. if isinstance(setting_type.description, str): # If it's a string, then the setting matches a single field in the DB, # and its name is the setting type's name attribute. - self.loaded_settings[setting_type.name] = setting_type.description, setting_type + self.loaded_settings[setting_type.name] = ( + setting_type.description, setting_type, type_hints[setting_type.name] + ) else: # Otherwise, the setting type works with compound settings. self.loaded_settings.update({ - subsetting: (description, setting_type) + subsetting: (description, setting_type, type_hints[subsetting]) for subsetting, description in setting_type.description.items() }) @@ -128,9 +131,14 @@ def collect_loaded_types(self) -> None: extra_fields_type = filter_type.extra_fields_type if not extra_fields_type: continue + type_hints = get_type_hints(extra_fields_type) # A class var with a `_description` suffix is expected per field name. self.loaded_filter_settings[filter_name] = { - field_name: (getattr(extra_fields_type, f"{field_name}_description", ""), extra_fields_type) + field_name: ( + getattr(extra_fields_type, f"{field_name}_description", ""), + extra_fields_type, + type_hints[field_name] + ) for field_name in extra_fields_type.__fields__ } @@ -173,6 +181,31 @@ async def bl_list(self, ctx: Context, list_name: Optional[str] = None) -> None: list_type, filter_list = result await self._send_list(ctx, filter_list, list_type) + @blocklist.command(name="add", aliases=("a",)) + async def bl_add( + self, + ctx: Context, + noui: Optional[Literal["noui"]], + list_name: Optional[str], + content: str, + *, + description_and_settings: Optional[str] = None + ) -> None: + """ + Add a blocked filter to the specified filter list. + + Unless `noui` is specified, a UI will be provided to edit the content, description, and settings + before confirmation. + + The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the + equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. + """ + result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name) + if result is None: + return + list_type, filter_list = result + await self._add_filter(ctx, noui, list_type, filter_list, content, description_and_settings) + # endregion # region: whitelist commands @@ -191,6 +224,31 @@ async def al_list(self, ctx: Context, list_name: Optional[str] = None) -> None: list_type, filter_list = result await self._send_list(ctx, filter_list, list_type) + @allowlist.command(name="add", aliases=("a",)) + async def al_add( + self, + ctx: Context, + noui: Optional[Literal["noui"]], + list_name: Optional[str], + content: str, + *, + description_and_settings: Optional[str] = None + ) -> None: + """ + Add an allowed filter to the specified filter list. + + Unless `noui` is specified, a UI will be provided to edit the content, description, and settings + before confirmation. + + The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the + equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. + """ + result = await self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name) + if result is None: + return + list_type, filter_list = result + await self._add_filter(ctx, noui, list_type, filter_list, content, description_and_settings) + # endregion # region: filter commands @@ -224,23 +282,16 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: for _, setting in settings.items(): overrides_values.update(to_serializable(setting.dict())) - # Combine them. It's done in this way to preserve field order, since the filter won't have all settings. - total_values = {} - for name, value in default_setting_values.items(): - if name not in overrides_values: - total_values[name] = value - else: - total_values[f"{name}*"] = overrides_values[name] - # Add the filter-specific settings. - if hasattr(filter_.extra_fields, "dict"): + if filter_.extra_fields_type: extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) - for name, value in filter_.extra_fields.dict().items(): - if name not in extra_fields_overrides: - total_values[f"{filter_.name}/{name}"] = value - else: - total_values[f"{filter_.name}/{name}*"] = value + else: + extra_fields_overrides = {} - embed = self._build_embed_from_dict(total_values) + all_settings_repr_dict = build_filter_repr_dict( + filter_list, list_type, type(filter_), overrides_values, extra_fields_overrides + ) + embed = Embed(colour=Colour.blue()) + populate_embed_from_dict(embed, all_settings_repr_dict) embed.description = f"`{filter_.content}`" if filter_.description: embed.description += f" - {filter_.description}" @@ -253,7 +304,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: @filter.command(name="list", aliases=("get",)) async def f_list( - self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None + self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None ) -> None: """List the contents of a specified list of filters.""" result = await self._resolve_list_type_and_name(ctx, list_type, list_name) @@ -282,6 +333,34 @@ async def f_describe(self, ctx: Context, filter_name: Optional[str]) -> None: embed.colour = Colour.blue() await ctx.send(embed=embed) + @filter.command(name="add", aliases=("a",)) + async def f_add( + self, + ctx: Context, + noui: Optional[Literal["noui"]], + list_type: Optional[list_type_converter], + list_name: Optional[str], + content: str, + *, + description_and_settings: Optional[str] = None + ) -> None: + """ + Add a filter to the specified filter list. + + Unless `noui` is specified, a UI will be provided to edit the content, description, and settings + before confirmation. + + The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the + equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. + + Example: `!filter add denied token "Scaleios is great" delete_messages=True send_alert=False` + """ + result = await self._resolve_list_type_and_name(ctx, list_type, list_name) + if result is None: + return + list_type, filter_list = result + await self._add_filter(ctx, noui, list_type, filter_list, content, description_and_settings) + @filter.group(aliases=("settings",)) async def setting(self, ctx: Context) -> None: """Group for settings-related commands.""" @@ -347,7 +426,8 @@ async def fl_describe( for _, setting in list_defaults[type_].items(): setting_values.update(to_serializable(setting.dict())) - embed = self._build_embed_from_dict(setting_values) + embed = Embed(colour=Colour.blue()) + populate_embed_from_dict(embed, setting_values) # Use the class's docstring, and ignore single newlines. embed.description = re.sub(r"(? Optional[tuple[Filter, FilterList, List if id_ in sublist: return sublist[id_], filter_list, list_type + async def _add_filter( + self, + ctx: Context, + noui: Optional[Literal["noui"]], + list_type: ListType, + filter_list: FilterList, + content: str, + description_and_settings: Optional[str] = None + ) -> None: + """Add a filter to the database.""" + description, settings, filter_settings = description_and_settings_converter( + filter_list.name, self.loaded_settings, self.loaded_filter_settings, description_and_settings + ) + filter_type = filter_list.get_filter_type(content) + + if noui: + await self._post_new_filter( + ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings + ) + + else: + embed = Embed(colour=Colour.blue()) + embed.description = f"`{content}`" if content else "*No content*" + if description: + embed.description += f" - {description}" + embed.set_author( + name=f"New Filter - {past_tense(list_type.name.lower())} {filter_list.name}".title()) + embed.set_footer(text=( + "Field names with an asterisk have values which override the defaults of the containing filter list. " + f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + )) + + view = filters_ui.SettingsEditView( + filter_list, + list_type, + filter_type, + content, + description, + settings, + filter_settings, + self.loaded_settings, + self.loaded_filter_settings, + ctx.author, + embed, + self._post_new_filter + ) + await ctx.send(embed=embed, reference=ctx.message, view=view) + @staticmethod - def _build_embed_from_dict(data: dict) -> Embed: - """Build a Discord embed by populating fields from the given dict.""" - embed = Embed(description="", colour=Colour.blue()) - for setting, value in data.items(): - if setting.startswith("_"): - continue - value = str(value) if value not in ("", None) else "-" - if len(value) > MAX_FIELD_SIZE: - value = value[:MAX_FIELD_SIZE] + " [...]" - embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) - return embed + async def _post_new_filter( + msg: Message, + filter_list: FilterList, + list_type: ListType, + filter_type: type[Filter], + content: str, + description: str | None, + settings: dict, + filter_settings: dict + ) -> None: + """POST the data of the new filter to the site API.""" + valid, error_msg = filter_type.validate_filter_settings(filter_settings) + if not valid: + raise BadArgument(f"Error while validating filter-specific settings: {error_msg}") + + list_id = filter_list.list_ids[list_type] + description = description or None + payload = { + "filter_list": list_id, "content": content, "description": description, + "additional_field": json.dumps(filter_settings), **settings + } + response = await bot.instance.api_client.post('bot/filter/filters', json=payload) + new_filter = filter_list.add_filter(response, list_type) + await msg.channel.send(f"✅ Added filter: {new_filter}", reference=msg) # endregion From c813e3fae3502920d62d2b4ca6d65d894cca939b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 30 Sep 2022 21:53:26 +0300 Subject: [PATCH 014/132] Add filter edit command --- bot/exts/filtering/filtering.py | 135 +++++++++++++++++++++++++++----- 1 file changed, 116 insertions(+), 19 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 7ffa121e13..8ff26795e4 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -2,7 +2,7 @@ import operator import re from collections import defaultdict -from functools import reduce +from functools import partial, reduce from typing import Literal, Optional, get_type_hints from discord import Colour, Embed, HTTPException, Message @@ -269,23 +269,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: return filter_, filter_list, list_type = result - # Get filter list settings - default_setting_values = {} - for type_ in ("actions", "validations"): - for _, setting in filter_list.defaults[list_type][type_].items(): - default_setting_values.update(to_serializable(setting.dict())) - - # Get the filter's overridden settings - overrides_values = {} - for settings in (filter_.actions, filter_.validations): - if settings: - for _, setting in settings.items(): - overrides_values.update(to_serializable(setting.dict())) - - if filter_.extra_fields_type: - extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) - else: - extra_fields_overrides = {} + overrides_values, extra_fields_overrides = self._filter_overrides(filter_) all_settings_repr_dict = build_filter_repr_dict( filter_list, list_type, type(filter_), overrides_values, extra_fields_overrides @@ -361,6 +345,76 @@ async def f_add( list_type, filter_list = result await self._add_filter(ctx, noui, list_type, filter_list, content, description_and_settings) + @filter.command(name="edit", aliases=("e",)) + async def f_edit( + self, + ctx: Context, + noui: Optional[Literal["noui"]], + filter_id: int, + *, + description_and_settings: Optional[str] = None + ) -> None: + """ + Edit a filter specified by its ID. + + Unless `noui` is specified, a UI will be provided to edit the content, description, and settings + before confirmation. + + The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the + equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. + + To edit the filter's content, use the UI. + """ + result = self._get_filter_by_id(filter_id) + if result is None: + await ctx.send(f":x: Could not find a filter with ID `{filter_id}`.") + return + filter_, filter_list, list_type = result + filter_type = type(filter_) + settings, filter_settings = self._filter_overrides(filter_) + description, new_settings, new_filter_settings = description_and_settings_converter( + filter_list.name, self.loaded_settings, self.loaded_filter_settings, description_and_settings + ) + + content = filter_.content + description = description or filter_.description + settings.update(new_settings) + filter_settings.update(new_filter_settings) + patch_func = partial(self._patch_filter, filter_id) + + if noui: + await patch_func( + ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings + ) + + else: + embed = Embed(colour=Colour.blue()) + embed.description = f"`{filter_.content}`" + if description: + embed.description += f" - {description}" + embed.set_author( + name=f"Filter #{filter_id} - {past_tense(list_type.name.lower())} {filter_list.name}".title()) + embed.set_footer(text=( + "Field names with an asterisk have values which override the defaults of the containing filter list. " + f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + )) + + view = filters_ui.SettingsEditView( + filter_list, + list_type, + filter_type, + content, + description, + settings, + filter_settings, + self.loaded_settings, + self.loaded_filter_settings, + ctx.author, + embed, + patch_func + ) + await ctx.send(embed=embed, reference=ctx.message, view=view) + @filter.group(aliases=("settings",)) async def setting(self, ctx: Context) -> None: """Group for settings-related commands.""" @@ -553,6 +607,22 @@ def _get_filter_by_id(self, id_: int) -> Optional[tuple[Filter, FilterList, List if id_ in sublist: return sublist[id_], filter_list, list_type + @staticmethod + def _filter_overrides(filter_: Filter) -> tuple[dict, dict]: + """Get the filter's overrides to the filter list settings and the extra fields settings.""" + overrides_values = {} + for settings in (filter_.actions, filter_.validations): + if settings: + for _, setting in settings.items(): + overrides_values.update(to_serializable(setting.dict())) + + if filter_.extra_fields_type: + extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) + else: + extra_fields_overrides = {} + + return overrides_values, extra_fields_overrides + async def _add_filter( self, ctx: Context, @@ -625,7 +695,34 @@ async def _post_new_filter( } response = await bot.instance.api_client.post('bot/filter/filters', json=payload) new_filter = filter_list.add_filter(response, list_type) - await msg.channel.send(f"✅ Added filter: {new_filter}", reference=msg) + await msg.reply(f"✅ Added filter: {new_filter}") + + @staticmethod + async def _patch_filter( + filter_id: int, + msg: Message, + filter_list: FilterList, + list_type: ListType, + filter_type: type[Filter], + content: str, + description: str | None, + settings: dict, + filter_settings: dict + ) -> None: + """PATCH the new data of the filter to the site API.""" + valid, error_msg = filter_type.validate_filter_settings(filter_settings) + if not valid: + raise BadArgument(f"Error while validating filter-specific settings: {error_msg}") + + list_id = filter_list.list_ids[list_type] + description = description or None + payload = { + "filter_list": list_id, "content": content, "description": description, + "additional_field": json.dumps(filter_settings), **settings + } + response = await bot.instance.api_client.patch(f'bot/filter/filters/{filter_id}', json=payload) + edited_filter = filter_list.add_filter(response, list_type) + await msg.reply(f"✅ Edited filter: {edited_filter}") # endregion From dbb078bd4d5d4d558138004b2d4b639b02c8106b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 30 Sep 2022 22:32:16 +0300 Subject: [PATCH 015/132] Add filter delete command --- bot/exts/filtering/filtering.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 8ff26795e4..1beb23ced4 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -415,6 +415,18 @@ async def f_edit( ) await ctx.send(embed=embed, reference=ctx.message, view=view) + @filter.command(name="delete", aliases=("d", "remove")) + async def f_delete(self, ctx: Context, filter_id: int) -> None: + """Delete the filter specified by its ID.""" + result = self._get_filter_by_id(filter_id) + if result is None: + await ctx.send(f":x: Could not find a filter with ID `{filter_id}`.") + return + filter_, filter_list, list_type = result + await bot.instance.api_client.delete(f'bot/filter/filters/{filter_id}') + filter_list.filter_lists[list_type].pop(filter_id) + await ctx.reply(f"✅ Deleted filter: {filter_}") + @filter.group(aliases=("settings",)) async def setting(self, ctx: Context) -> None: """Group for settings-related commands.""" From 4483b7cdd55fe46ddc6f35e10343f44f4531144d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 30 Sep 2022 23:24:01 +0300 Subject: [PATCH 016/132] Voice ban -> voice mute in auto-infractions This is a remnant after the last rebase. --- .../filtering/_settings_types/infraction_and_notification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/infraction_and_notification.py index c1fc007605..4fcf2aa656 100644 --- a/bot/exts/filtering/_settings_types/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/infraction_and_notification.py @@ -19,7 +19,7 @@ class Infraction(Enum): BAN = auto() KICK = auto() MUTE = auto() - VOICE_BAN = auto() + VOICE_MUTE = auto() SUPERSTAR = auto() WARNING = auto() WATCH = auto() From 72e164c38fed8d02fbe58412cf3a6de6e38aec09 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 30 Sep 2022 23:45:49 +0300 Subject: [PATCH 017/132] Split actions and validations to their own packcages This is a purely aesthetic choice. Additionally fixes a small bug where a missing entry type would repeatedly invoke a warning on cog load. --- bot/exts/filtering/_settings.py | 2 +- bot/exts/filtering/_settings_types/__init__.py | 9 ++------- bot/exts/filtering/_settings_types/actions/__init__.py | 8 ++++++++ .../_settings_types/{ => actions}/delete_messages.py | 0 .../{ => actions}/infraction_and_notification.py | 0 bot/exts/filtering/_settings_types/{ => actions}/ping.py | 0 .../_settings_types/{ => actions}/send_alert.py | 0 .../filtering/_settings_types/validations/__init__.py | 8 ++++++++ .../_settings_types/{ => validations}/bypass_roles.py | 0 .../_settings_types/{ => validations}/channel_scope.py | 0 .../_settings_types/{ => validations}/enabled.py | 0 .../_settings_types/{ => validations}/filter_dm.py | 0 tests/bot/exts/filtering/test_settings_entries.py | 8 ++++---- 13 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 bot/exts/filtering/_settings_types/actions/__init__.py rename bot/exts/filtering/_settings_types/{ => actions}/delete_messages.py (100%) rename bot/exts/filtering/_settings_types/{ => actions}/infraction_and_notification.py (100%) rename bot/exts/filtering/_settings_types/{ => actions}/ping.py (100%) rename bot/exts/filtering/_settings_types/{ => actions}/send_alert.py (100%) create mode 100644 bot/exts/filtering/_settings_types/validations/__init__.py rename bot/exts/filtering/_settings_types/{ => validations}/bypass_roles.py (100%) rename bot/exts/filtering/_settings_types/{ => validations}/channel_scope.py (100%) rename bot/exts/filtering/_settings_types/{ => validations}/enabled.py (100%) rename bot/exts/filtering/_settings_types/{ => validations}/filter_dm.py (100%) diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index f88b26ee32..cbd682d6db 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -31,7 +31,7 @@ def create_settings( action_data[entry_name] = entry_data elif entry_name in settings_types["ValidationEntry"]: validation_data[entry_name] = entry_data - else: + elif entry_name not in _already_warned: log.warning( f"A setting named {entry_name} was loaded from the database, but no matching class." ) diff --git a/bot/exts/filtering/_settings_types/__init__.py b/bot/exts/filtering/_settings_types/__init__.py index 620290cb27..61b5737d41 100644 --- a/bot/exts/filtering/_settings_types/__init__.py +++ b/bot/exts/filtering/_settings_types/__init__.py @@ -1,10 +1,5 @@ -from os.path import dirname - -from bot.exts.filtering._settings_types.settings_entry import ActionEntry, ValidationEntry -from bot.exts.filtering._utils import subclasses_in_package - -action_types = subclasses_in_package(dirname(__file__), f"{__name__}.", ActionEntry) -validation_types = subclasses_in_package(dirname(__file__), f"{__name__}.", ValidationEntry) +from bot.exts.filtering._settings_types.actions import action_types +from bot.exts.filtering._settings_types.validations import validation_types settings_types = { "ActionEntry": {settings_type.name: settings_type for settings_type in action_types}, diff --git a/bot/exts/filtering/_settings_types/actions/__init__.py b/bot/exts/filtering/_settings_types/actions/__init__.py new file mode 100644 index 0000000000..a8175b9764 --- /dev/null +++ b/bot/exts/filtering/_settings_types/actions/__init__.py @@ -0,0 +1,8 @@ +from os.path import dirname + +from bot.exts.filtering._settings_types.settings_entry import ActionEntry +from bot.exts.filtering._utils import subclasses_in_package + +action_types = subclasses_in_package(dirname(__file__), f"{__name__}.", ActionEntry) + +__all__ = [action_types] diff --git a/bot/exts/filtering/_settings_types/delete_messages.py b/bot/exts/filtering/_settings_types/actions/delete_messages.py similarity index 100% rename from bot/exts/filtering/_settings_types/delete_messages.py rename to bot/exts/filtering/_settings_types/actions/delete_messages.py diff --git a/bot/exts/filtering/_settings_types/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py similarity index 100% rename from bot/exts/filtering/_settings_types/infraction_and_notification.py rename to bot/exts/filtering/_settings_types/actions/infraction_and_notification.py diff --git a/bot/exts/filtering/_settings_types/ping.py b/bot/exts/filtering/_settings_types/actions/ping.py similarity index 100% rename from bot/exts/filtering/_settings_types/ping.py rename to bot/exts/filtering/_settings_types/actions/ping.py diff --git a/bot/exts/filtering/_settings_types/send_alert.py b/bot/exts/filtering/_settings_types/actions/send_alert.py similarity index 100% rename from bot/exts/filtering/_settings_types/send_alert.py rename to bot/exts/filtering/_settings_types/actions/send_alert.py diff --git a/bot/exts/filtering/_settings_types/validations/__init__.py b/bot/exts/filtering/_settings_types/validations/__init__.py new file mode 100644 index 0000000000..5c44e8b276 --- /dev/null +++ b/bot/exts/filtering/_settings_types/validations/__init__.py @@ -0,0 +1,8 @@ +from os.path import dirname + +from bot.exts.filtering._settings_types.settings_entry import ValidationEntry +from bot.exts.filtering._utils import subclasses_in_package + +validation_types = subclasses_in_package(dirname(__file__), f"{__name__}.", ValidationEntry) + +__all__ = [validation_types] diff --git a/bot/exts/filtering/_settings_types/bypass_roles.py b/bot/exts/filtering/_settings_types/validations/bypass_roles.py similarity index 100% rename from bot/exts/filtering/_settings_types/bypass_roles.py rename to bot/exts/filtering/_settings_types/validations/bypass_roles.py diff --git a/bot/exts/filtering/_settings_types/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py similarity index 100% rename from bot/exts/filtering/_settings_types/channel_scope.py rename to bot/exts/filtering/_settings_types/validations/channel_scope.py diff --git a/bot/exts/filtering/_settings_types/enabled.py b/bot/exts/filtering/_settings_types/validations/enabled.py similarity index 100% rename from bot/exts/filtering/_settings_types/enabled.py rename to bot/exts/filtering/_settings_types/validations/enabled.py diff --git a/bot/exts/filtering/_settings_types/filter_dm.py b/bot/exts/filtering/_settings_types/validations/filter_dm.py similarity index 100% rename from bot/exts/filtering/_settings_types/filter_dm.py rename to bot/exts/filtering/_settings_types/validations/filter_dm.py diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py index d18861bd6f..8dba5cb26a 100644 --- a/tests/bot/exts/filtering/test_settings_entries.py +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -1,12 +1,12 @@ import unittest from bot.exts.filtering._filter_context import Event, FilterContext -from bot.exts.filtering._settings_types.bypass_roles import RoleBypass -from bot.exts.filtering._settings_types.channel_scope import ChannelScope -from bot.exts.filtering._settings_types.filter_dm import FilterDM -from bot.exts.filtering._settings_types.infraction_and_notification import ( +from bot.exts.filtering._settings_types.actions.infraction_and_notification import ( Infraction, InfractionAndNotification, superstar ) +from bot.exts.filtering._settings_types.validations.bypass_roles import RoleBypass +from bot.exts.filtering._settings_types.validations.channel_scope import ChannelScope +from bot.exts.filtering._settings_types.validations.filter_dm import FilterDM from tests.helpers import MockCategoryChannel, MockDMChannel, MockMember, MockMessage, MockRole, MockTextChannel From 7d46b1ed1fdec2a052c147a89f69354469bbfd18 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 00:14:21 +0300 Subject: [PATCH 018/132] Fix tests --- .../validations/bypass_roles.py | 2 +- .../_settings_types/validations/filter_dm.py | 2 +- tests/bot/exts/filtering/test_settings.py | 2 +- .../exts/filtering/test_settings_entries.py | 182 ++++-------------- tests/bot/rules/test_mentions.py | 131 ------------- tests/helpers.py | 6 +- 6 files changed, 44 insertions(+), 281 deletions(-) delete mode 100644 tests/bot/rules/test_mentions.py diff --git a/bot/exts/filtering/_settings_types/validations/bypass_roles.py b/bot/exts/filtering/_settings_types/validations/bypass_roles.py index a5c18cffc9..c1e6f885db 100644 --- a/bot/exts/filtering/_settings_types/validations/bypass_roles.py +++ b/bot/exts/filtering/_settings_types/validations/bypass_roles.py @@ -15,7 +15,7 @@ class RoleBypass(ValidationEntry): bypass_roles: set[Union[int, str]] - @validator("bypass_roles", each_item=True) + @validator("bypass_roles", pre=True, each_item=True) @classmethod def maybe_cast_to_int(cls, role: str) -> Union[int, str]: """If the string is alphanumeric, cast it to int.""" diff --git a/bot/exts/filtering/_settings_types/validations/filter_dm.py b/bot/exts/filtering/_settings_types/validations/filter_dm.py index 93022320f6..b9e566253a 100644 --- a/bot/exts/filtering/_settings_types/validations/filter_dm.py +++ b/bot/exts/filtering/_settings_types/validations/filter_dm.py @@ -14,4 +14,4 @@ class FilterDM(ValidationEntry): def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered even if it was triggered in DMs.""" - return hasattr(ctx.channel, "guild") or self.filter_dm + return ctx.channel.guild is not None or self.filter_dm diff --git a/tests/bot/exts/filtering/test_settings.py b/tests/bot/exts/filtering/test_settings.py index ac21a5d477..5a289c1cf1 100644 --- a/tests/bot/exts/filtering/test_settings.py +++ b/tests/bot/exts/filtering/test_settings.py @@ -11,7 +11,7 @@ def test_create_settings_returns_none_for_empty_data(self): """`create_settings` should return a tuple of two Nones when passed an empty dict.""" result = create_settings({}) - self.assertEquals(result, (None, None)) + self.assertEqual(result, (None, None)) def test_unrecognized_entry_makes_a_warning(self): """When an unrecognized entry name is passed to `create_settings`, it should be added to `_already_warned`.""" diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py index 8dba5cb26a..34b155d6b6 100644 --- a/tests/bot/exts/filtering/test_settings_entries.py +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -1,9 +1,7 @@ import unittest from bot.exts.filtering._filter_context import Event, FilterContext -from bot.exts.filtering._settings_types.actions.infraction_and_notification import ( - Infraction, InfractionAndNotification, superstar -) +from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction, InfractionAndNotification from bot.exts.filtering._settings_types.validations.bypass_roles import RoleBypass from bot.exts.filtering._settings_types.validations.channel_scope import ChannelScope from bot.exts.filtering._settings_types.validations.filter_dm import FilterDM @@ -23,7 +21,7 @@ def test_role_bypass_is_off_for_user_without_roles(self): """The role bypass should trigger when a user has no roles.""" member = MockMember() self.ctx.author = member - bypass_entry = RoleBypass(["123"]) + bypass_entry = RoleBypass(bypass_roles=["123"]) result = bypass_entry.triggers_on(self.ctx) @@ -43,7 +41,7 @@ def test_role_bypass_is_on_for_a_user_with_the_right_role(self): user_roles = [MockRole(id=role_id) for role_id in user_role_ids] member = MockMember(roles=user_roles) self.ctx.author = member - bypass_entry = RoleBypass(bypasses) + bypass_entry = RoleBypass(bypass_roles=bypasses) result = bypass_entry.triggers_on(self.ctx) @@ -52,7 +50,7 @@ def test_role_bypass_is_on_for_a_user_with_the_right_role(self): def test_context_doesnt_trigger_for_empty_channel_scope(self): """A filter is enabled for all channels by default.""" channel = MockTextChannel() - scope = ChannelScope({"disabled_channels": None, "disabled_categories": None, "enabled_channels": None}) + scope = ChannelScope(disabled_channels=None, disabled_categories=None, enabled_channels=None) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -62,7 +60,7 @@ def test_context_doesnt_trigger_for_empty_channel_scope(self): def test_context_doesnt_trigger_for_disabled_channel(self): """A filter shouldn't trigger if it's been disabled in the channel.""" channel = MockTextChannel(id=123) - scope = ChannelScope({"disabled_channels": ["123"], "disabled_categories": None, "enabled_channels": None}) + scope = ChannelScope(disabled_channels=["123"], disabled_categories=None, enabled_channels=None) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -72,9 +70,7 @@ def test_context_doesnt_trigger_for_disabled_channel(self): def test_context_doesnt_trigger_in_disabled_category(self): """A filter shouldn't trigger if it's been disabled in the category.""" channel = MockTextChannel(category=MockCategoryChannel(id=456)) - scope = ChannelScope({ - "disabled_channels": None, "disabled_categories": ["456"], "enabled_channels": None - }) + scope = ChannelScope(disabled_channels=None, disabled_categories=["456"], enabled_channels=None) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -84,7 +80,7 @@ def test_context_doesnt_trigger_in_disabled_category(self): def test_context_triggers_in_enabled_channel_in_disabled_category(self): """A filter should trigger in an enabled channel even if it's been disabled in the category.""" channel = MockTextChannel(id=123, category=MockCategoryChannel(id=234)) - scope = ChannelScope({"disabled_channels": None, "disabled_categories": ["234"], "enabled_channels": ["123"]}) + scope = ChannelScope(disabled_channels=None, disabled_categories=["234"], enabled_channels=["123"]) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -102,7 +98,7 @@ def test_filtering_dms_when_necessary(self): for apply_in_dms, channel, expected in cases: with self.subTest(apply_in_dms=apply_in_dms, channel=channel): - filter_dms = FilterDM(apply_in_dms) + filter_dms = FilterDM(filter_dm=apply_in_dms) self.ctx.channel = channel result = filter_dms.triggers_on(self.ctx) @@ -111,162 +107,60 @@ def test_filtering_dms_when_necessary(self): def test_infraction_merge_of_same_infraction_type(self): """When both infractions are of the same type, the one with the longer duration wins.""" - infraction1 = InfractionAndNotification({ - "infraction_type": "mute", - "infraction_reason": "hi", - "infraction_duration": 10, - "dm_content": "how", - "dm_embed": "what is" - }) - infraction2 = InfractionAndNotification({ - "infraction_type": "mute", - "infraction_reason": "there", - "infraction_duration": 20, - "dm_content": "are you", - "dm_embed": "your name" - }) + infraction1 = InfractionAndNotification( + infraction_type="MUTE", + infraction_reason="hi", + infraction_duration=10, + dm_content="how", + dm_embed="what is" + ) + infraction2 = InfractionAndNotification( + infraction_type="MUTE", + infraction_reason="there", + infraction_duration=20, + dm_content="are you", + dm_embed="your name" + ) result = infraction1 | infraction2 self.assertDictEqual( - result.to_dict(), + result.dict(), { "infraction_type": Infraction.MUTE, "infraction_reason": "there", "infraction_duration": 20.0, "dm_content": "are you", "dm_embed": "your name", - "_superstar": None } ) def test_infraction_merge_of_different_infraction_types(self): """If there are two different infraction types, the one higher up the hierarchy should be picked.""" - infraction1 = InfractionAndNotification({ - "infraction_type": "mute", - "infraction_reason": "hi", - "infraction_duration": 20, - "dm_content": "", - "dm_embed": "" - }) - infraction2 = InfractionAndNotification({ - "infraction_type": "ban", - "infraction_reason": "", - "infraction_duration": 10, - "dm_content": "there", - "dm_embed": "" - }) + infraction1 = InfractionAndNotification( + infraction_type="MUTE", + infraction_reason="hi", + infraction_duration=20, + dm_content="", + dm_embed="" + ) + infraction2 = InfractionAndNotification( + infraction_type="BAN", + infraction_reason="", + infraction_duration=10, + dm_content="there", + dm_embed="" + ) result = infraction1 | infraction2 self.assertDictEqual( - result.to_dict(), + result.dict(), { "infraction_type": Infraction.BAN, "infraction_reason": "", "infraction_duration": 10.0, "dm_content": "there", "dm_embed": "", - "_superstar": None - } - ) - - def test_infraction_merge_with_a_superstar(self): - """If there is a superstar infraction, it should be added to a separate field.""" - infraction1 = InfractionAndNotification({ - "infraction_type": "mute", - "infraction_reason": "hi", - "infraction_duration": 20, - "dm_content": "there", - "dm_embed": "" - }) - infraction2 = InfractionAndNotification({ - "infraction_type": "superstar", - "infraction_reason": "hello", - "infraction_duration": 10, - "dm_content": "you", - "dm_embed": "" - }) - - result = infraction1 | infraction2 - - self.assertDictEqual( - result.to_dict(), - { - "infraction_type": Infraction.MUTE, - "infraction_reason": "hi", - "infraction_duration": 20.0, - "dm_content": "there", - "dm_embed": "", - "_superstar": superstar("hello", 10.0) - } - ) - - def test_merge_two_superstar_infractions(self): - """When two superstar infractions are merged, the infraction type remains a superstar.""" - infraction1 = InfractionAndNotification({ - "infraction_type": "superstar", - "infraction_reason": "hi", - "infraction_duration": 20, - "dm_content": "", - "dm_embed": "" - }) - infraction2 = InfractionAndNotification({ - "infraction_type": "superstar", - "infraction_reason": "", - "infraction_duration": 10, - "dm_content": "there", - "dm_embed": "" - }) - - result = infraction1 | infraction2 - - self.assertDictEqual( - result.to_dict(), - { - "infraction_type": Infraction.SUPERSTAR, - "infraction_reason": "hi", - "infraction_duration": 20.0, - "dm_content": "", - "dm_embed": "", - "_superstar": None - } - ) - - def test_merge_a_voiceban_and_a_superstar_with_another_superstar(self): - """An infraction with a superstar merged with a superstar should combine under `_superstar`.""" - infraction1 = InfractionAndNotification({ - "infraction_type": "voice ban", - "infraction_reason": "hi", - "infraction_duration": 20, - "dm_content": "hello", - "dm_embed": "" - }) - infraction2 = InfractionAndNotification({ - "infraction_type": "superstar", - "infraction_reason": "bla", - "infraction_duration": 10, - "dm_content": "there", - "dm_embed": "" - }) - infraction3 = InfractionAndNotification({ - "infraction_type": "superstar", - "infraction_reason": "blabla", - "infraction_duration": 20, - "dm_content": "there", - "dm_embed": "" - }) - - result = infraction1 | infraction2 | infraction3 - - self.assertDictEqual( - result.to_dict(), - { - "infraction_type": Infraction.VOICE_BAN, - "infraction_reason": "hi", - "infraction_duration": 20, - "dm_content": "hello", - "dm_embed": "", - "_superstar": superstar("blabla", 20) } ) diff --git a/tests/bot/rules/test_mentions.py b/tests/bot/rules/test_mentions.py deleted file mode 100644 index e1f9049175..0000000000 --- a/tests/bot/rules/test_mentions.py +++ /dev/null @@ -1,131 +0,0 @@ -from typing import Iterable, Optional - -import discord - -from bot.rules import mentions -from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMember, MockMessage, MockMessageReference - - -def make_msg( - author: str, - total_user_mentions: int, - total_bot_mentions: int = 0, - *, - reference: Optional[MockMessageReference] = None -) -> MockMessage: - """Makes a message from `author` with `total_user_mentions` user mentions and `total_bot_mentions` bot mentions.""" - user_mentions = [MockMember() for _ in range(total_user_mentions)] - bot_mentions = [MockMember(bot=True) for _ in range(total_bot_mentions)] - - mentions = user_mentions + bot_mentions - if reference is not None: - # For the sake of these tests we assume that all references are mentions. - mentions.append(reference.resolved.author) - msg_type = discord.MessageType.reply - else: - msg_type = discord.MessageType.default - - return MockMessage(author=author, mentions=mentions, reference=reference, type=msg_type) - - -class TestMentions(RuleTest): - """Tests applying the `mentions` antispam rule.""" - - def setUp(self): - self.apply = mentions.apply - self.config = { - "max": 2, - "interval": 10, - } - - async def test_mentions_within_limit(self): - """Messages with an allowed amount of mentions.""" - cases = ( - [make_msg("bob", 0)], - [make_msg("bob", 2)], - [make_msg("bob", 1), make_msg("bob", 1)], - [make_msg("bob", 1), make_msg("alice", 2)], - ) - - await self.run_allowed(cases) - - async def test_mentions_exceeding_limit(self): - """Messages with a higher than allowed amount of mentions.""" - cases = ( - DisallowedCase( - [make_msg("bob", 3)], - ("bob",), - 3, - ), - DisallowedCase( - [make_msg("alice", 2), make_msg("alice", 0), make_msg("alice", 1)], - ("alice",), - 3, - ), - DisallowedCase( - [make_msg("bob", 2), make_msg("alice", 3), make_msg("bob", 2)], - ("bob",), - 4, - ), - DisallowedCase( - [make_msg("bob", 3, 1)], - ("bob",), - 3, - ), - DisallowedCase( - [make_msg("bob", 3, reference=MockMessageReference())], - ("bob",), - 3, - ), - DisallowedCase( - [make_msg("bob", 3, reference=MockMessageReference(reference_author_is_bot=True))], - ("bob",), - 3 - ) - ) - - await self.run_disallowed(cases) - - async def test_ignore_bot_mentions(self): - """Messages with an allowed amount of mentions, also containing bot mentions.""" - cases = ( - [make_msg("bob", 0, 3)], - [make_msg("bob", 2, 1)], - [make_msg("bob", 1, 2), make_msg("bob", 1, 2)], - [make_msg("bob", 1, 5), make_msg("alice", 2, 5)] - ) - - await self.run_allowed(cases) - - async def test_ignore_reply_mentions(self): - """Messages with an allowed amount of mentions in the content, also containing reply mentions.""" - cases = ( - [ - make_msg("bob", 2, reference=MockMessageReference()) - ], - [ - make_msg("bob", 2, reference=MockMessageReference(reference_author_is_bot=True)) - ], - [ - make_msg("bob", 2, reference=MockMessageReference()), - make_msg("bob", 0, reference=MockMessageReference()) - ], - [ - make_msg("bob", 2, reference=MockMessageReference(reference_author_is_bot=True)), - make_msg("bob", 0, reference=MockMessageReference(reference_author_is_bot=True)) - ] - ) - - await self.run_allowed(cases) - - def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: - last_message = case.recent_messages[0] - return tuple( - msg - for msg in case.recent_messages - if msg.author == last_message.author - ) - - def get_report(self, case: DisallowedCase) -> str: - return f"sent {case.n_violations} mentions in {self.config['interval']}s" diff --git a/tests/helpers.py b/tests/helpers.py index 28a8e40a7c..35a8a71f73 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -393,15 +393,15 @@ def __init__(self, **kwargs) -> None: class MockDMChannel(CustomMockMixin, unittest.mock.Mock, HashableMixin): """ - A MagicMock subclass to mock TextChannel objects. + A MagicMock subclass to mock DMChannel objects. - Instances of this class will follow the specifications of `discord.TextChannel` instances. For + Instances of this class will follow the specifications of `discord.DMChannel` instances. For more information, see the `MockGuild` docstring. """ spec_set = dm_channel_instance def __init__(self, **kwargs) -> None: - default_kwargs = {'id': next(self.discord_id), 'recipient': MockUser(), "me": MockUser()} + default_kwargs = {'id': next(self.discord_id), 'recipient': MockUser(), "me": MockUser(), 'guild': None} super().__init__(**collections.ChainMap(kwargs, default_kwargs)) From 5cf4a752162bc336929b34c37c99cca675127521 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 00:16:22 +0300 Subject: [PATCH 019/132] Correct logging --- bot/exts/filtering/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 1beb23ced4..841ae4e99e 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -70,7 +70,7 @@ async def cog_load(self) -> None: try: self.webhook = await self.bot.fetch_webhook(Webhooks.filters) except HTTPException: - log.error(f"Failed to fetch incidents webhook with ID `{Webhooks.incidents}`.") + log.error(f"Failed to fetch filters webhook with ID `{Webhooks.filters}`.") self.collect_loaded_types() From b07e27638c85864e260df5d011c6f746a0f10948 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 00:24:07 +0300 Subject: [PATCH 020/132] Use try-except instead of is_digit in bypass_roles --- .../filtering/_settings_types/validations/bypass_roles.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bot/exts/filtering/_settings_types/validations/bypass_roles.py b/bot/exts/filtering/_settings_types/validations/bypass_roles.py index c1e6f885db..193b07e133 100644 --- a/bot/exts/filtering/_settings_types/validations/bypass_roles.py +++ b/bot/exts/filtering/_settings_types/validations/bypass_roles.py @@ -18,10 +18,11 @@ class RoleBypass(ValidationEntry): @validator("bypass_roles", pre=True, each_item=True) @classmethod def maybe_cast_to_int(cls, role: str) -> Union[int, str]: - """If the string is alphanumeric, cast it to int.""" - if role.isdigit(): + """If the string is numeric, cast it to int.""" + try: return int(role) - return role + except ValueError: + return role def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered on this user given their roles.""" From f09f84274de172aaf1db15a294eb179846c77c44 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 00:25:28 +0300 Subject: [PATCH 021/132] Keep sequences as lists in editing UI Sequences such as sets are not serializable, and it's going to be converted to the right type when loaded to the cog anyway. --- bot/exts/filtering/_ui.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index d40d4d41e7..2d0e7d8911 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -337,10 +337,9 @@ async def free_input(self, interaction: Interaction, button: discord.ui.Button) @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green) async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: """Send the final value to the embed editor.""" - final_value = self.type_(self.stored_value) # Edit first, it might time out otherwise. await interaction.response.edit_message(content="✅ Edit confirmed", view=None) - await self.update_callback(setting_name=self.setting_name, setting_value=final_value) + await self.update_callback(setting_name=self.setting_name, setting_value=self.stored_value) self.stop() @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red) From e485da8dc7b08e1ee65f022a686a98a4f162e75a Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 00:48:20 +0300 Subject: [PATCH 022/132] Properly remove items from sequence edit Some items might not be stored as strings. --- bot/exts/filtering/_ui.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 2d0e7d8911..676afbbf6b 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -302,7 +302,14 @@ def __init__(self, setting_name: str, starting_value: list, type_: type, update_ async def apply_removal(self, interaction: Interaction, select: discord.ui.Select) -> None: """Remove an item from the list.""" - self.stored_value.remove(select.values[0]) + # The value might not be stored as a string. + _i = len(self.stored_value) + for _i, element in enumerate(self.stored_value): + if str(element) == select.values[0]: + break + if _i != len(self.stored_value): + self.stored_value.pop(_i) + select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] if not self.stored_value: self.remove_item(self.removal_select) From 8bb8b45f0bce5bd2a8b69e5372d6c065e9dbb2bd Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 00:49:00 +0300 Subject: [PATCH 023/132] Correctly handle DMs --- .../filtering/_settings_types/actions/delete_messages.py | 6 ++++-- .../filtering/_settings_types/validations/channel_scope.py | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/delete_messages.py b/bot/exts/filtering/_settings_types/actions/delete_messages.py index 710cb0ed8f..d1ddf82412 100644 --- a/bot/exts/filtering/_settings_types/actions/delete_messages.py +++ b/bot/exts/filtering/_settings_types/actions/delete_messages.py @@ -22,9 +22,11 @@ async def action(self, ctx: FilterContext) -> None: if not self.delete_messages or ctx.event not in (Event.MESSAGE, Event.MESSAGE_EDIT): return + if not ctx.message.guild: + return + with suppress(NotFound): - if ctx.message.guild: - await ctx.message.delete() + await ctx.message.delete() ctx.action_descriptions.append("deleted") def __or__(self, other: ActionEntry): diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index fd5206b814..be10611991 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -49,6 +49,10 @@ def triggers_on(self, ctx: FilterContext) -> bool: If the channel is explicitly enabled, it bypasses the set disabled channels and categories. """ channel = ctx.channel + + if channel.guild is None: # This is a DM channel, outside the scope of this setting. + return True + enabled_id = ( channel.id in self.enabled_channels or ( From ff25bd0af481b8113f468bc6a48d7374aa3a27d8 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 09:25:27 +0300 Subject: [PATCH 024/132] Fix patching removed override --- bot/exts/filtering/filtering.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 841ae4e99e..925c40da51 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -380,7 +380,7 @@ async def f_edit( description = description or filter_.description settings.update(new_settings) filter_settings.update(new_filter_settings) - patch_func = partial(self._patch_filter, filter_id) + patch_func = partial(self._patch_filter, filter_) if noui: await patch_func( @@ -711,7 +711,7 @@ async def _post_new_filter( @staticmethod async def _patch_filter( - filter_id: int, + filter_: Filter, msg: Message, filter_list: FilterList, list_type: ListType, @@ -726,13 +726,19 @@ async def _patch_filter( if not valid: raise BadArgument(f"Error while validating filter-specific settings: {error_msg}") + # If the setting is not in `settings`, the override was either removed, or there wasn't one in the first place. + for current_settings in (filter_.actions, filter_.validations): + if current_settings: + for _, setting_entry in current_settings.items(): + settings.update({setting: None for setting in setting_entry.dict() if setting not in settings}) + list_id = filter_list.list_ids[list_type] description = description or None payload = { "filter_list": list_id, "content": content, "description": description, "additional_field": json.dumps(filter_settings), **settings } - response = await bot.instance.api_client.patch(f'bot/filter/filters/{filter_id}', json=payload) + response = await bot.instance.api_client.patch(f'bot/filter/filters/{filter_.id}', json=payload) edited_filter = filter_list.add_filter(response, list_type) await msg.reply(f"✅ Edited filter: {edited_filter}") From 37bab917fdb17a435ca625bce1cc3ff76bdec802 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 10:01:21 +0300 Subject: [PATCH 025/132] Fix channel_scope checking IDs and names separately --- .../validations/channel_scope.py | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index be10611991..9fb4e2ba74 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -37,9 +37,10 @@ def init_if_sequence_none(cls, sequence: list[str]) -> list[str]: @classmethod def maybe_cast_items(cls, channel_or_category: str) -> Union[str, int]: """Cast to int each value in each sequence if it is alphanumeric.""" - if channel_or_category.isdigit(): + try: return int(channel_or_category) - return channel_or_category + except ValueError: + return channel_or_category def triggers_on(self, ctx: FilterContext) -> bool: """ @@ -53,18 +54,10 @@ def triggers_on(self, ctx: FilterContext) -> bool: if channel.guild is None: # This is a DM channel, outside the scope of this setting. return True - enabled_id = ( - channel.id in self.enabled_channels - or ( - channel.id not in self.disabled_channels - and (not channel.category or channel.category.id not in self.disabled_categories) - ) + enabled_channel = channel.id in self.enabled_channels or channel.name in self.enabled_channels + disabled_channel = channel.id in self.disabled_channels or channel.name in self.disabled_channels + disabled_category = channel.category and ( + channel.category.id in self.disabled_categories or channel.category.name in self.disabled_categories ) - enabled_name = ( - channel.name in self.enabled_channels - or ( - channel.name not in self.disabled_channels - and (not channel.category or channel.category.name not in self.disabled_categories) - ) - ) - return enabled_id and enabled_name + + return enabled_channel or (not disabled_channel and not disabled_category) From fb1af0030abacb57d2ff06470fd19ce2d51358fb Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 10:01:31 +0300 Subject: [PATCH 026/132] Stress that a filter was triggered in DM --- bot/exts/filtering/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 925c40da51..8d9c62ce36 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -539,7 +539,7 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi if ctx.channel.guild: triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}" else: - triggered_in = "**DM**" + triggered_in = "**Triggered in:** :warning:**DM**:warning:" filters = [] for filter_list, list_message in triggered_filters.items(): From fbe3a2e24ac26881c389ae268c273278e110319d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 20:53:34 +0300 Subject: [PATCH 027/132] Bring back enabled categories, remove redundant validators There needs to be a way to only enable a filter in a specific category, so this setting now fulfills that role. Disabled channels can be used to disable a filter in a specific channel within the category. Additionally the validators which maybe convert to int are now removed. As long as the int is the first in the Union, it will happen anyway. --- .../validations/bypass_roles.py | 10 ------ .../validations/channel_scope.py | 35 ++++++++++--------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/bot/exts/filtering/_settings_types/validations/bypass_roles.py b/bot/exts/filtering/_settings_types/validations/bypass_roles.py index 193b07e133..d42e6407c6 100644 --- a/bot/exts/filtering/_settings_types/validations/bypass_roles.py +++ b/bot/exts/filtering/_settings_types/validations/bypass_roles.py @@ -1,7 +1,6 @@ from typing import ClassVar, Union from discord import Member -from pydantic import validator from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ValidationEntry @@ -15,15 +14,6 @@ class RoleBypass(ValidationEntry): bypass_roles: set[Union[int, str]] - @validator("bypass_roles", pre=True, each_item=True) - @classmethod - def maybe_cast_to_int(cls, role: str) -> Union[int, str]: - """If the string is numeric, cast it to int.""" - try: - return int(role) - except ValueError: - return role - def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered on this user given their roles.""" if not isinstance(ctx.author, Member): diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index 9fb4e2ba74..deae55dfc4 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -11,19 +11,28 @@ class ChannelScope(ValidationEntry): name: ClassVar[str] = "channel_scope" description: ClassVar[str] = { - "disabled_channels": "A list of channel IDs or channel names. The filter will not trigger in these channels.", + "disabled_channels": ( + "A list of channel IDs or channel names. " + "The filter will not trigger in these channels even if the category is expressly enabled." + ), "disabled_categories": ( "A list of category IDs or category names. The filter will not trigger in these categories." ), "enabled_channels": ( "A list of channel IDs or channel names. " - "The filter can trigger in these channels even if the category is disabled." + "The filter can trigger in these channels even if the category is disabled or not expressly enabled." + ), + "enabled_categories": ( + "A list of category IDs or category names. " + "If the list is not empty, filters will trigger only in channels of these categories, " + "unless the channel is expressly disabled." ) } - disabled_channels: set[Union[str, int]] - disabled_categories: set[Union[str, int]] - enabled_channels: set[Union[str, int]] + disabled_channels: set[Union[int, str]] + disabled_categories: set[Union[int, str]] + enabled_channels: set[Union[int, str]] + enabled_categories: set[Union[int, str]] @validator("*", pre=True) @classmethod @@ -33,15 +42,6 @@ def init_if_sequence_none(cls, sequence: list[str]) -> list[str]: return [] return sequence - @validator("*", each_item=True) - @classmethod - def maybe_cast_items(cls, channel_or_category: str) -> Union[str, int]: - """Cast to int each value in each sequence if it is alphanumeric.""" - try: - return int(channel_or_category) - except ValueError: - return channel_or_category - def triggers_on(self, ctx: FilterContext) -> bool: """ Return whether the filter should be triggered in the given channel. @@ -51,13 +51,16 @@ def triggers_on(self, ctx: FilterContext) -> bool: """ channel = ctx.channel - if channel.guild is None: # This is a DM channel, outside the scope of this setting. + if not hasattr(channel, "category"): # This is not a guild channel, outside the scope of this setting. return True enabled_channel = channel.id in self.enabled_channels or channel.name in self.enabled_channels disabled_channel = channel.id in self.disabled_channels or channel.name in self.disabled_channels + enabled_category = channel.category and (not self.enabled_categories or ( + channel.category.id in self.enabled_categories or channel.category.name in self.enabled_categories + )) disabled_category = channel.category and ( channel.category.id in self.disabled_categories or channel.category.name in self.disabled_categories ) - return enabled_channel or (not disabled_channel and not disabled_category) + return enabled_channel or (enabled_category and not disabled_channel and not disabled_category) From 39d3a777cd89c560ccc964648ca78e0f8545e0dd Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 20:54:08 +0300 Subject: [PATCH 028/132] Add a warning if an added filter has content identicalto others --- bot/exts/filtering/filtering.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 8d9c62ce36..b6153487c8 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -683,6 +683,17 @@ async def _add_filter( ) await ctx.send(embed=embed, reference=ctx.message, view=view) + @staticmethod + def _identical_filters_message(content: str, filter_list: FilterList, list_type: ListType) -> str: + """Returns all the filters in the list with content identical to the content supplied.""" + duplicates = [f for f in filter_list.filter_lists.get(list_type, {}).values() if f.content == content] + msg = "" + if duplicates: + msg = f"\n:warning: The filter(s) #{', #'.join(str(dup.id) for dup in duplicates)} have the same content. " + msg += "Please make sure this is intentional." + + return msg + @staticmethod async def _post_new_filter( msg: Message, @@ -706,8 +717,9 @@ async def _post_new_filter( "additional_field": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.post('bot/filter/filters', json=payload) + extra_msg = Filtering._identical_filters_message(content, filter_list, list_type) new_filter = filter_list.add_filter(response, list_type) - await msg.reply(f"✅ Added filter: {new_filter}") + await msg.reply(f"✅ Added filter: {new_filter}" + extra_msg) @staticmethod async def _patch_filter( @@ -739,8 +751,9 @@ async def _patch_filter( "additional_field": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.patch(f'bot/filter/filters/{filter_.id}', json=payload) + extra_msg = Filtering._identical_filters_message(content, filter_list, list_type) edited_filter = filter_list.add_filter(response, list_type) - await msg.reply(f"✅ Edited filter: {edited_filter}") + await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) # endregion From 29f3aede4039ec6cc29a6317a58141159de0d1cc Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 21:03:17 +0300 Subject: [PATCH 029/132] Show settings in sorted order in UI --- bot/exts/filtering/_ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 676afbbf6b..8640b48c45 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -427,7 +427,7 @@ def __init__( add_select = CustomCallbackSelect( self._prompt_new_override, placeholder="Select a setting to edit", - options=[SelectOption(label=name) for name in self.type_per_setting_name], + options=[SelectOption(label=name) for name in sorted(self.type_per_setting_name)], row=1 ) self.add_item(add_select) @@ -436,7 +436,7 @@ def __init__( remove_select = CustomCallbackSelect( self._remove_override, placeholder="Select an override to remove", - options=[SelectOption(label=name) for name in override_names], + options=[SelectOption(label=name) for name in sorted(override_names)], row=2 ) if remove_select.options: From 83d4e4074c249ffcf8a9810e483dd42f1acc2dbb Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 22:29:38 +0300 Subject: [PATCH 030/132] Prettify post/patch filter error --- bot/exts/filtering/_ui.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 8640b48c45..94eb86c3bd 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -34,6 +34,7 @@ MAX_MODAL_LABEL_LENGTH = 45 # Max number of items in a select MAX_SELECT_ITEMS = 25 +MAX_EMBED_DESCRIPTION = 4000 T = TypeVar('T') @@ -485,9 +486,10 @@ async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> self.filter_settings_overrides ) except ResponseCodeError as e: - await interaction.message.channel.send(f"An error occurred: ```{e}```", reference=interaction.message) - - self.stop() + await interaction.message.reply(embed=format_response_error(e)) + await interaction.message.edit(view=self) + else: + self.stop() @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red, row=4) async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: @@ -689,3 +691,19 @@ def description_and_settings_converter( raise BadArgument(e) return description, settings, filter_settings + + +def format_response_error(e: ResponseCodeError) -> Embed: + """Format the response error into an embed.""" + description = "" + if "non_field_errors" in e.response_json: + non_field_errors = e.response_json.pop("non_field_errors") + description += "\n".join(f"• {error}" for error in non_field_errors) + "\n" + for field, errors in e.response_json.items(): + description += "\n".join(f"• {field} - {error}" for error in errors) + "\n" + description = description.strip() + if len(description) > MAX_EMBED_DESCRIPTION: + description = description[:MAX_EMBED_DESCRIPTION] + "[...]" + + embed = Embed(colour=discord.Colour.red(), title="Oops...", description=description) + return embed From 68c47c4fa6b83a447580d6e412cd728f472c8ebd Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 1 Oct 2022 22:34:40 +0300 Subject: [PATCH 031/132] Handle invalid UI edits --- bot/exts/filtering/_ui.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 94eb86c3bd..7030bf07b2 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -577,11 +577,15 @@ async def update_embed( self.embed.clear_fields() new_view = self.copy() - if isinstance(interaction_or_msg, discord.Interaction): - await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) + try: + if isinstance(interaction_or_msg, discord.Interaction): + await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) + else: + await interaction_or_msg.edit(embed=self.embed, view=new_view) + except discord.errors.HTTPException: # Various error such as embed description being too long. + pass else: - await interaction_or_msg.edit(embed=self.embed, view=new_view) - self.stop() + self.stop() async def edit_setting_override(self, interaction: Interaction, setting_name: str, override_value: Any) -> None: """ From f91963fd11330bca62cfdbf6c8f2030e14787829 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 3 Oct 2022 20:51:13 +0300 Subject: [PATCH 032/132] Add filter content processing before posting/patching --- bot/exts/filtering/_filter_lists/domain.py | 2 +- bot/exts/filtering/_filters/domain.py | 15 ++++++++++++++ bot/exts/filtering/_filters/extension.py | 11 +++++++++++ bot/exts/filtering/_filters/filter.py | 9 +++++++++ bot/exts/filtering/_filters/invite.py | 23 ++++++++++++++++++++++ bot/exts/filtering/_filters/token.py | 13 ++++++++++++ bot/exts/filtering/_ui.py | 7 ++++++- bot/exts/filtering/filtering.py | 14 ++++++++++--- 8 files changed, 89 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index 6e12dad411..c407108caa 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -16,7 +16,7 @@ if typing.TYPE_CHECKING: from bot.exts.filtering.filtering import Filtering -URL_RE = re.compile(r"https?://([^\s]+)", flags=re.IGNORECASE) +URL_RE = re.compile(r"https?://(\S+)", flags=re.IGNORECASE) class DomainsList(FilterList): diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 9f5f974134..2a0cd3c313 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -1,3 +1,4 @@ +import re from typing import ClassVar, Optional import tldextract @@ -6,6 +7,8 @@ from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter +URL_RE = re.compile(r"(?:https?://)?(\S+?)[\\/]*", flags=re.IGNORECASE) + class ExtraDomainSettings(BaseModel): """Extra settings for how domains should be matched in a message.""" @@ -43,3 +46,15 @@ def triggered_on(self, ctx: FilterContext) -> bool: ctx.notification_domain = self.content return not self.extra_fields.exact or self.content == found_url return False + + @classmethod + async def process_content(cls, content: str) -> str: + """ + Process the content into a form which will work with the filtering. + + A ValueError should be raised if the content can't be used. + """ + match = URL_RE.fullmatch(content) + if not match or not match.group(1): + raise ValueError(f"`{content}` is not a URL.") + return match.group(1) diff --git a/bot/exts/filtering/_filters/extension.py b/bot/exts/filtering/_filters/extension.py index 1a2ab86174..926a6a2fb9 100644 --- a/bot/exts/filtering/_filters/extension.py +++ b/bot/exts/filtering/_filters/extension.py @@ -14,3 +14,14 @@ class ExtensionFilter(Filter): def triggered_on(self, ctx: FilterContext) -> bool: """Searches for an attachment extension in the context content, given as a set of extensions.""" return self.content in ctx.content + + @classmethod + async def process_content(cls, content: str) -> str: + """ + Process the content into a form which will work with the filtering. + + A ValueError should be raised if the content can't be used. + """ + if not content.startswith("."): + content = f".{content}" + return content diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 92393871ad..957957d834 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -48,6 +48,15 @@ def validate_filter_settings(cls, extra_fields: dict) -> tuple[bool, Optional[st else: return True, None + @classmethod + async def process_content(cls, content: str) -> str: + """ + Process the content into a form which will work with the filtering. + + A ValueError should be raised if the content can't be used. + """ + return content + def __str__(self) -> str: """A string representation of the filter.""" string = f"#{self.id}. `{self.content}`" diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py index 5a99248331..e0f4695206 100644 --- a/bot/exts/filtering/_filters/invite.py +++ b/bot/exts/filtering/_filters/invite.py @@ -1,3 +1,7 @@ +from botcore.utils.regex import DISCORD_INVITE +from discord import NotFound + +import bot from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter @@ -18,3 +22,22 @@ def __init__(self, filter_data: dict): def triggered_on(self, ctx: FilterContext) -> bool: """Searches for a guild ID in the context content, given as a set of IDs.""" return self.content in ctx.content + + @classmethod + async def process_content(cls, content: str) -> str: + """ + Process the content into a form which will work with the filtering. + + A ValueError should be raised if the content can't be used. + """ + match = DISCORD_INVITE.fullmatch(content) + if not match or not match.group("invite"): + raise ValueError(f"`{content}` is not a valid Discord invite.") + invite_code = match.group("invite") + try: + invite = await bot.instance.fetch_invite(invite_code) + except NotFound: + raise ValueError(f"`{invite_code}` is not a valid Discord invite code.") + if not invite.guild: + raise ValueError("Did you just try to add a group DM?") + return str(invite.guild.id) diff --git a/bot/exts/filtering/_filters/token.py b/bot/exts/filtering/_filters/token.py index c955b269ba..a4c646c5a7 100644 --- a/bot/exts/filtering/_filters/token.py +++ b/bot/exts/filtering/_filters/token.py @@ -18,3 +18,16 @@ def triggered_on(self, ctx: FilterContext) -> bool: ctx.matches.append(match[0]) return True return False + + @classmethod + async def process_content(cls, content: str) -> str: + """ + Process the content into a form which will work with the filtering. + + A ValueError should be raised if the content can't be used. + """ + try: + re.compile(content) + except re.error as e: + raise ValueError(str(e)) + return content diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 7030bf07b2..12d1f213d0 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -488,6 +488,11 @@ async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> except ResponseCodeError as e: await interaction.message.reply(embed=format_response_error(e)) await interaction.message.edit(view=self) + except ValueError as e: + await interaction.message.reply( + embed=Embed(colour=discord.Colour.red(), title="Bad Content", description=str(e)) + ) + await interaction.message.edit(view=self) else: self.stop() @@ -556,7 +561,7 @@ async def update_embed( # Update the embed with the new content and/or description. self.embed.description = f"`{content}`" if content else "*No content*" - if description is not None and description is not self._REMOVE: + if description and description is not self._REMOVE: self.embed.description += f" - {description}" if setting_name: diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index b6153487c8..cdbb521fbe 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -651,9 +651,12 @@ async def _add_filter( filter_type = filter_list.get_filter_type(content) if noui: - await self._post_new_filter( - ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings - ) + try: + await self._post_new_filter( + ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings + ) + except ValueError as e: + raise BadArgument(str(e)) else: embed = Embed(colour=Colour.blue()) @@ -710,6 +713,8 @@ async def _post_new_filter( if not valid: raise BadArgument(f"Error while validating filter-specific settings: {error_msg}") + content = await filter_type.process_content(content) + list_id = filter_list.list_ids[list_type] description = description or None payload = { @@ -738,6 +743,9 @@ async def _patch_filter( if not valid: raise BadArgument(f"Error while validating filter-specific settings: {error_msg}") + if content != filter_.content: + content = await filter_type.process_content(content) + # If the setting is not in `settings`, the override was either removed, or there wasn't one in the first place. for current_settings in (filter_.actions, filter_.validations): if current_settings: From 6db78fa911bd529badd67db5a8356c06bc648140 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 6 Oct 2022 20:26:10 +0300 Subject: [PATCH 033/132] Fix bug with setting domain to notify Unfortunately the filter doesn't know its full effect, only its overrides, so it might not know whether it's going to delete the message or not. This commit prevents errors, but might lead to unwanted behavior if there are two domains in a message, one that causes deletion and one that doesn't, and the second one gets evaluated last. This is basically a 'I hope no one notices' fix until I can think of something better. In practice this shouldn't be an issue. --- bot/exts/filtering/_filters/domain.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 2a0cd3c313..00a9a886f7 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -39,11 +39,7 @@ def triggered_on(self, ctx: FilterContext) -> bool: for found_url in ctx.content: if self.content in found_url and tldextract.extract(found_url).registered_domain == domain: ctx.matches.append(self.content) - if ( - ("delete_messages" in self.actions and self.actions.get("delete_messages").delete_messages) - or not ctx.notification_domain - ): # Override this field only if this filter causes deletion. - ctx.notification_domain = self.content + ctx.notification_domain = self.content return not self.extra_fields.exact or self.content == found_url return False From f12cc721737bf0bd64ddd7605cdc51ce06020b93 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 6 Oct 2022 21:57:18 +0300 Subject: [PATCH 034/132] domain/exact -> domain/subdomains The original plan was to have a field which only matches subdomains and not the domain itself, I got confused when I wrote the /exact field. This fixes a bug where the matches would be updated even if it didn't meet the criteria of the extra field. Also fixes some issues in the UI dealing with filter overrides. --- bot/exts/filtering/_filters/domain.py | 17 +++++++++++------ bot/exts/filtering/_ui.py | 8 +++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 00a9a886f7..eed2b67213 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -1,5 +1,6 @@ import re from typing import ClassVar, Optional +from urllib.parse import urlparse import tldextract from pydantic import BaseModel @@ -14,11 +15,11 @@ class ExtraDomainSettings(BaseModel): """Extra settings for how domains should be matched in a message.""" exact_description: ClassVar[str] = ( - "A boolean. If True, will match the filter content exactly, and won't trigger for subdomains and subpaths." + "A boolean. If True, will will only trigger for subdomains and subpaths, and not for the domain itself." ) - # whether to match the filter content exactly, or to trigger for subdomains and subpaths as well. - exact: Optional[bool] = False + # Whether to trigger only for subdomains and subpaths, and not the specified domain itself. + subdomains: Optional[bool] = False class DomainFilter(Filter): @@ -26,7 +27,7 @@ class DomainFilter(Filter): A filter which looks for a specific domain given by URL. The schema (http, https) does not need to be included in the filter. - Will also match subdomains unless set otherwise. + Will also match subdomains. """ name = "domain" @@ -37,10 +38,14 @@ def triggered_on(self, ctx: FilterContext) -> bool: domain = tldextract.extract(self.content).registered_domain for found_url in ctx.content: - if self.content in found_url and tldextract.extract(found_url).registered_domain == domain: + extract = tldextract.extract(found_url) + if self.content in found_url and extract.registered_domain == domain: + if self.extra_fields.subdomains: + if not extract.subdomain and not urlparse(f"https://{found_url}").path: + return False ctx.matches.append(self.content) ctx.notification_domain = self.content - return not self.extra_fields.exact or self.content == found_url + return True return False @classmethod diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 12d1f213d0..ec20510832 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -433,7 +433,9 @@ def __init__( ) self.add_item(add_select) - override_names = list(settings_overrides) + list(filter_settings_overrides) + override_names = ( + list(settings_overrides) + [f"{filter_list.name}/{setting}" for setting in filter_settings_overrides] + ) remove_select = CustomCallbackSelect( self._remove_override, placeholder="Select an override to remove", @@ -568,13 +570,13 @@ async def update_embed( # Find the right dictionary to update. if "/" in setting_name: filter_name, setting_name = setting_name.split("/", maxsplit=1) - dict_to_edit = self.filter_settings_overrides[filter_name] + dict_to_edit = self.filter_settings_overrides else: dict_to_edit = self.settings_overrides # Update the setting override value or remove it if setting_value is not self._REMOVE: dict_to_edit[setting_name] = setting_value - else: + elif setting_name in dict_to_edit: del dict_to_edit[setting_name] # This is inefficient, but otherwise the selects go insane if the user attempts to edit the same setting From 7e227f6a71458bd6b8126332d1d9da73bfa7bfeb Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 7 Oct 2022 16:53:52 +0300 Subject: [PATCH 035/132] Make auto-infraction actually work --- .../actions/infraction_and_notification.py | 80 +++++++++++++++---- bot/exts/moderation/infraction/infractions.py | 2 +- 2 files changed, 66 insertions(+), 16 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 4fcf2aa656..d4095590e0 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -1,18 +1,52 @@ +from dataclasses import dataclass from datetime import timedelta from enum import Enum, auto from typing import ClassVar import arrow -from discord import Colour, Embed +import discord.abc +from discord import Colour, Embed, Member, User from discord.errors import Forbidden from pydantic import validator -import bot +import bot as bot_module from bot.constants import Channels, Guild from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +@dataclass +class FakeContext: + """ + A class representing a context-like object that can be sent to infraction commands. + + The goal is to be able to apply infractions without depending on the existence of a message or an interaction + (which are the two ways to create a Context), e.g. in API events which aren't message-driven, or in custom filtering + events. + """ + + channel: discord.abc.Messageable + bot: bot_module.bot.Bot | None = None + guild: discord.Guild | None = None + author: discord.Member | discord.User | None = None + me: discord.Member | None = None + + def __post_init__(self): + """Initialize the missing information.""" + if not self.bot: + self.bot = bot_module.instance + if not self.guild: + self.guild = self.bot.get_guild(Guild.id) + if not self.me: + self.me = self.guild.me + if not self.author: + self.author = self.me + + async def send(self, *args, **kwargs) -> discord.Message: + """A wrapper for channel.send.""" + return await self.channel.send(*args, **kwargs) + + class Infraction(Enum): """An enumeration of infraction types. The lower the value, the higher it is on the hierarchy.""" @@ -28,6 +62,30 @@ class Infraction(Enum): def __str__(self) -> str: return self.name + async def invoke( + self, + user: Member | User, + channel: discord.abc.Messageable | None, + duration: float | None = None, + reason: str | None = None + ) -> None: + """Invokes the command matching the infraction name.""" + alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) + if not channel: + channel = alerts_channel + + command_name = self.name.lower() + command = bot_module.instance.get_command(command_name) + if not command: + await alerts_channel.send(f":warning: Could not apply {command_name} to {user.mention}: command not found.") + + ctx = FakeContext(channel) + if self.name in ("KICK", "WARNING", "WATCH", "NOTE"): + await command(ctx, user, reason=reason) + else: + duration = arrow.utcnow() + timedelta(seconds=duration) if duration else None + await command(ctx, user, duration, reason=reason) + class InfractionAndNotification(ActionEntry): """ @@ -87,21 +145,13 @@ async def action(self, ctx: FilterContext) -> None: except Forbidden: ctx.action_descriptions.append("notified (failed)") - msg_ctx = await bot.instance.get_context(ctx.message) - msg_ctx.guild = bot.instance.get_guild(Guild.id) - msg_ctx.author = ctx.author - msg_ctx.channel = ctx.channel - if self.infraction_type is not None: if self.infraction_type == Infraction.BAN or not hasattr(ctx.channel, "guild"): - msg_ctx.channel = bot.instance.get_channel(Channels.mod_alerts) - msg_ctx.command = bot.instance.get_command(self.infraction_type.name.lower()) - await msg_ctx.invoke( - msg_ctx.command, - ctx.author, - arrow.utcnow() + timedelta(seconds=self.infraction_duration) - if self.infraction_duration is not None else None, - reason=self.infraction_reason + infrac_channel = None + else: + infrac_channel = ctx.channel + await self.infraction_type.invoke( + ctx.author, infrac_channel, self.infraction_duration, self.infraction_reason ) ctx.action_descriptions.append(self.infraction_type.name.lower()) diff --git a/bot/exts/moderation/infraction/infractions.py b/bot/exts/moderation/infraction/infractions.py index 999f9ba7f6..1ef43a9453 100644 --- a/bot/exts/moderation/infraction/infractions.py +++ b/bot/exts/moderation/infraction/infractions.py @@ -79,7 +79,7 @@ async def action() -> None: # region: Permanent infractions - @command() + @command(aliases=("warning",)) async def warn(self, ctx: Context, user: UnambiguousMemberOrUser, *, reason: t.Optional[str] = None) -> None: """Warn a user for the given reason.""" if not isinstance(user, Member): From 19a660ff9399d0d8ebd6a802a7c45595839e6077 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 7 Oct 2022 18:40:22 +0300 Subject: [PATCH 036/132] Add more spacing in mod alert --- bot/exts/filtering/filtering.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index cdbb521fbe..11a3948f3a 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -537,9 +537,9 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi embed.set_thumbnail(url=ctx.author.display_avatar.url) triggered_by = f"**Triggered by:** {format_user(ctx.author)}" if ctx.channel.guild: - triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}" + triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}\n" else: - triggered_in = "**Triggered in:** :warning:**DM**:warning:" + triggered_in = "**Triggered in:** :warning:**DM**:warning:\n" filters = [] for filter_list, list_message in triggered_filters.items(): @@ -548,8 +548,8 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi filters = "\n".join(filters) matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) - actions = "**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") - content = f"**[Original Content]({ctx.message.jump_url})**: {escape_markdown(ctx.content)}" + actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") + content = f"**[Original Content]({ctx.message.jump_url})**:\n{escape_markdown(ctx.content)}" embed_content = "\n".join( part for part in (triggered_by, triggered_in, filters, matches, actions, content) if part From b0df8b0bbe5cbcc7168339564c52a941c1b44756 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 7 Oct 2022 18:44:45 +0300 Subject: [PATCH 037/132] Prevent the duplicate filter warning from mentioning itself --- bot/exts/filtering/filtering.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 11a3948f3a..5f42e2cabd 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -687,9 +687,12 @@ async def _add_filter( await ctx.send(embed=embed, reference=ctx.message, view=view) @staticmethod - def _identical_filters_message(content: str, filter_list: FilterList, list_type: ListType) -> str: + def _identical_filters_message(content: str, filter_list: FilterList, list_type: ListType, filter_: Filter) -> str: """Returns all the filters in the list with content identical to the content supplied.""" - duplicates = [f for f in filter_list.filter_lists.get(list_type, {}).values() if f.content == content] + duplicates = [ + f for f in filter_list.filter_lists.get(list_type, {}).values() + if f.content == content and f.id != filter_.id + ] msg = "" if duplicates: msg = f"\n:warning: The filter(s) #{', #'.join(str(dup.id) for dup in duplicates)} have the same content. " @@ -722,8 +725,8 @@ async def _post_new_filter( "additional_field": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.post('bot/filter/filters', json=payload) - extra_msg = Filtering._identical_filters_message(content, filter_list, list_type) new_filter = filter_list.add_filter(response, list_type) + extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, new_filter) await msg.reply(f"✅ Added filter: {new_filter}" + extra_msg) @staticmethod @@ -759,8 +762,8 @@ async def _patch_filter( "additional_field": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.patch(f'bot/filter/filters/{filter_.id}', json=payload) - extra_msg = Filtering._identical_filters_message(content, filter_list, list_type) edited_filter = filter_list.add_filter(response, list_type) + extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, edited_filter) await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) # endregion From ffcef56ff0e22208917a299e2caec4bf3428b02e Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 8 Oct 2022 01:16:22 +0300 Subject: [PATCH 038/132] Add infraction channel setting --- .../actions/infraction_and_notification.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index d4095590e0..4ec06ef4c6 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -65,14 +65,16 @@ def __str__(self) -> str: async def invoke( self, user: Member | User, - channel: discord.abc.Messageable | None, - duration: float | None = None, - reason: str | None = None + channel: int | None, + duration: float | None, + reason: str | None ) -> None: """Invokes the command matching the infraction name.""" alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) if not channel: channel = alerts_channel + else: + channel = bot_module.instance.get_channel(channel) command_name = self.name.lower() command = bot_module.instance.get_command(command_name) @@ -99,12 +101,12 @@ class InfractionAndNotification(ActionEntry): "infraction_type": ( "The type of infraction to issue when the filter triggers, or 'NONE'. " "If two infractions are triggered for the same message, " - "the harsher one will be applied (by type or duration). " - "Superstars will be triggered even if there is a harsher infraction.\n\n" + "the harsher one will be applied (by type or duration).\n\n" "Valid infraction types in order of harshness: " ) + ", ".join(infraction.name for infraction in Infraction), "infraction_duration": "How long the infraction should last for in seconds, or 'None' for permanent.", "infraction_reason": "The reason delivered with the infraction.", + "infraction_channel": "The channel ID in which to invoke the infraction (and send the confirmation message).", "dm_content": "The contents of a message to be DMed to the offending user.", "dm_embed": "The contents of the embed to be DMed to the offending user." } @@ -114,6 +116,7 @@ class InfractionAndNotification(ActionEntry): infraction_type: Infraction | None infraction_reason: str | None infraction_duration: float | None + infraction_channel: int | None @validator("infraction_type", pre=True) @classmethod @@ -146,12 +149,8 @@ async def action(self, ctx: FilterContext) -> None: ctx.action_descriptions.append("notified (failed)") if self.infraction_type is not None: - if self.infraction_type == Infraction.BAN or not hasattr(ctx.channel, "guild"): - infrac_channel = None - else: - infrac_channel = ctx.channel await self.infraction_type.invoke( - ctx.author, infrac_channel, self.infraction_duration, self.infraction_reason + ctx.author, self.infraction_channel, self.infraction_duration, self.infraction_reason ) ctx.action_descriptions.append(self.infraction_type.name.lower()) From bf02ac9c3691b25ffb486ec1625d528f1b83092d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 8 Oct 2022 11:45:42 +0300 Subject: [PATCH 039/132] Fix pings validation --- bot/exts/filtering/_settings_types/actions/ping.py | 4 ++-- .../filtering/_settings_types/validations/channel_scope.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/ping.py b/bot/exts/filtering/_settings_types/actions/ping.py index 0bfc12809b..85590478cb 100644 --- a/bot/exts/filtering/_settings_types/actions/ping.py +++ b/bot/exts/filtering/_settings_types/actions/ping.py @@ -26,9 +26,9 @@ class Ping(ActionEntry): guild_pings: set[str] dm_pings: set[str] - @validator("*") + @validator("*", pre=True) @classmethod - def init_sequence_if_none(cls, pings: list[str]) -> list[str]: + def init_sequence_if_none(cls, pings: list[str] | None) -> list[str]: """Initialize an empty sequence if the value is None.""" if pings is None: return [] diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index deae55dfc4..ae6c24c97a 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -36,7 +36,7 @@ class ChannelScope(ValidationEntry): @validator("*", pre=True) @classmethod - def init_if_sequence_none(cls, sequence: list[str]) -> list[str]: + def init_if_sequence_none(cls, sequence: list[str] | None) -> list[str]: """Initialize an empty sequence if the value is None.""" if sequence is None: return [] From ed7fd34b200ef4267f3735e770ac4b0d9b037559 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 8 Oct 2022 15:45:39 +0300 Subject: [PATCH 040/132] Ignore overrides that are equal to their defaults If an override is added that is equal to the default, it is ignored instead. If an override is read from a Filter object that is equal to its default, it is ignored instead. This is done because otherwise when a setting shares an entry with an overriden setting (e.g infraction_type with infraction_duration), there's no way of knowing whether it was truly overridden, since None values are not preserved for filters on loading (that is because the same pydantic models are used for both the filter and filter-list settings). This seems like an acceptable trade-off since overrides which are equal to their defaults don't have a practical use. Addiotionally, this commit fixes not being able to set an empty string as an override to a non-empty default. --- .../filtering/_filter_lists/filter_list.py | 12 ++++- bot/exts/filtering/_settings.py | 7 +++ .../_settings_types/settings_entry.py | 2 +- bot/exts/filtering/_ui.py | 48 ++++++++++++++----- bot/exts/filtering/_utils.py | 16 +++++++ bot/exts/filtering/filtering.py | 28 +++++++---- 6 files changed, 90 insertions(+), 23 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index c34f46878b..a620131924 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,6 +1,6 @@ from abc import abstractmethod from enum import Enum -from typing import Optional, Type +from typing import Any, Optional, Type from discord.ext.commands import BadArgument @@ -71,6 +71,16 @@ def add_filter(self, filter_data: dict, list_type: ListType) -> Filter: else: return new_filter + def default(self, list_type: ListType, setting: str) -> Any: + """Get the default value of a specific setting.""" + missing = object() + value = self.defaults[list_type]["actions"].get_setting(setting, missing) + if value is missing: + value = self.defaults[list_type]["validations"].get_setting(setting, missing) + if value is missing: + raise ValueError(f"Could find a setting named {setting}.") + return value + @abstractmethod def get_filter_type(self, content: str) -> Type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index cbd682d6db..7b09e3c520 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -107,6 +107,13 @@ def get(self, key: str, default: Optional[Any] = None) -> entry_type: """Get the entry matching the key, or fall back to the default value if the key is missing.""" return self._entries.get(key, default) + def get_setting(self, key: str, default: Optional[Any] = None) -> Any: + """Get the setting matching the key, or fall back to the default value if the key is missing.""" + for entry in self._entries.values(): + if hasattr(entry, key): + return getattr(entry, key) + return default + @classmethod def create(cls, settings_data: dict, *, keep_empty: bool = False) -> Optional[Settings]: """ diff --git a/bot/exts/filtering/_settings_types/settings_entry.py b/bot/exts/filtering/_settings_types/settings_entry.py index 2b3b030a05..5a7e41cac1 100644 --- a/bot/exts/filtering/_settings_types/settings_entry.py +++ b/bot/exts/filtering/_settings_types/settings_entry.py @@ -33,7 +33,7 @@ def create(cls, entry_data: Optional[dict[str, Any]], *, keep_empty: bool = Fals """ if entry_data is None: return None - if not keep_empty and hasattr(entry_data, "values") and not any(value for value in entry_data.values()): + if not keep_empty and hasattr(entry_data, "values") and all(value is None for value in entry_data.values()): return None if not isinstance(entry_data, dict): diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index ec20510832..8bfcded773 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -15,7 +15,7 @@ from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType from bot.exts.filtering._filters.filter import Filter -from bot.exts.filtering._utils import to_serializable +from bot.exts.filtering._utils import repr_equals, to_serializable from bot.log import get_logger log = get_logger(__name__) @@ -115,7 +115,7 @@ def build_filter_repr_dict( # Add overrides. It's done in this way to preserve field order, since the filter won't have all settings. total_values = {} for name, value in default_setting_values.items(): - if name not in settings_overrides: + if name not in settings_overrides or repr_equals(settings_overrides[name], value): total_values[name] = value else: total_values[f"{name}*"] = settings_overrides[name] @@ -124,7 +124,7 @@ def build_filter_repr_dict( if filter_type.extra_fields_type: # This iterates over the default values of the extra fields model. for name, value in filter_type.extra_fields_type().dict().items(): - if name not in extra_fields_overrides: + if name not in extra_fields_overrides or repr_equals(extra_fields_overrides[name], value): total_values[f"{filter_type.name}/{name}"] = value else: total_values[f"{filter_type.name}/{name}*"] = value @@ -248,7 +248,7 @@ def __init__(self, setting_name: str, required: bool, type_: type, update_callba async def on_submit(self, interaction: Interaction) -> None: """Update the setting with the new value in the embed.""" try: - value = self.type_(self.setting_input.value) or None + value = self.type_(self.setting_input.value) except (ValueError, TypeError): await interaction.response.send_message( f"Could not process the input value for `{self.setting_name}`.", ephemeral=True @@ -318,6 +318,10 @@ async def apply_removal(self, interaction: Interaction, select: discord.ui.Selec async def apply_addition(self, interaction: Interaction, item: str) -> None: """Add an item to the list.""" + if item in self.stored_value: # Ignore duplicates + await interaction.response.defer() + return + self.stored_value.append(item) self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] if len(self.stored_value) == 1: @@ -326,7 +330,7 @@ async def apply_addition(self, interaction: Interaction, item: str) -> None: async def apply_edit(self, interaction: Interaction, new_list: str) -> None: """Change the contents of the list.""" - self.stored_value = new_list.split(",") + self.stored_value = list(set(new_list.split(","))) self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] if len(self.stored_value) == 1: self.add_item(self.removal_select) @@ -571,11 +575,17 @@ async def update_embed( if "/" in setting_name: filter_name, setting_name = setting_name.split("/", maxsplit=1) dict_to_edit = self.filter_settings_overrides + default_value = self.filter_type.extra_fields_type().dict()[setting_name] else: dict_to_edit = self.settings_overrides + default_value = self.filter_list.default(self.list_type, setting_name) # Update the setting override value or remove it if setting_value is not self._REMOVE: - dict_to_edit[setting_name] = setting_value + if not repr_equals(setting_value, default_value): + dict_to_edit[setting_name] = setting_value + # If there's already an override, remove it, since the new value is the same as the default. + elif setting_name in dict_to_edit: + del dict_to_edit[setting_name] elif setting_name in dict_to_edit: del dict_to_edit[setting_name] @@ -657,7 +667,12 @@ def _parse_value(value: str, type_: type[T]) -> T: def description_and_settings_converter( - list_name: str, loaded_settings: dict, loaded_filter_settings: dict, input_data: str + filter_list: FilterList, + list_type: ListType, + filter_type: type[Filter], + loaded_settings: dict, + loaded_filter_settings: dict, + input_data: str ) -> tuple[str, dict[str, Any], dict[str, Any]]: """Parse a string representing a possible description and setting overrides, and validate the setting names.""" if not input_data: @@ -679,25 +694,32 @@ def description_and_settings_converter( filter_settings = {} for setting, _ in list(settings.items()): if setting not in loaded_settings: + # It's a filter setting if "/" in setting: setting_list_name, filter_setting_name = setting.split("/", maxsplit=1) - if setting_list_name.lower() != list_name.lower(): + if setting_list_name.lower() != filter_list.name.lower(): raise BadArgument( - f"A setting for a {setting_list_name!r} filter was provided, but the list name is {list_name!r}" + f"A setting for a {setting_list_name!r} filter was provided, " + f"but the list name is {filter_list.name!r}" ) - if filter_setting_name not in loaded_filter_settings[list_name]: + if filter_setting_name not in loaded_filter_settings[filter_list.name]: raise BadArgument(f"{setting!r} is not a recognized setting.") - type_ = loaded_filter_settings[list_name][filter_setting_name][2] + type_ = loaded_filter_settings[filter_list.name][filter_setting_name][2] try: - filter_settings[filter_setting_name] = _parse_value(settings.pop(setting), type_) + parsed_value = _parse_value(settings.pop(setting), type_) + if not repr_equals(parsed_value, getattr(filter_type.extra_fields_type(), filter_setting_name)): + filter_settings[filter_setting_name] = parsed_value except (TypeError, ValueError) as e: raise BadArgument(e) else: raise BadArgument(f"{setting!r} is not a recognized setting.") + # It's a filter list setting else: type_ = loaded_settings[setting][2] try: - settings[setting] = _parse_value(settings.pop(setting), type_) + parsed_value = _parse_value(settings.pop(setting), type_) + if not repr_equals(parsed_value, filter_list.default(list_type, setting)): + settings[setting] = parsed_value except (TypeError, ValueError) as e: raise BadArgument(e) diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index 158f1e7bd2..438c22d416 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -66,6 +66,22 @@ def to_serializable(item: Any) -> Union[bool, int, float, str, list, dict, None] return str(item) +def repr_equals(override: Any, default: Any) -> bool: + """Return whether the override and the default have the same representation.""" + if override is None: # It's not an override + return True + + override_is_sequence = isinstance(override, (tuple, list, set)) + default_is_sequence = isinstance(default, (tuple, list, set)) + if override_is_sequence != default_is_sequence: # One is a sequence and the other isn't. + return False + if override_is_sequence: + if len(override) != len(default): + return False + return all(str(item1) == str(item2) for item1, item2 in zip(set(override), set(default))) + return str(override) == str(default) + + class FieldRequiring(ABC): """A mixin class that can force its concrete subclasses to set a value for specific class attributes.""" diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 5f42e2cabd..aa90d1600b 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -21,7 +21,7 @@ from bot.exts.filtering._ui import ( ArgumentCompletionView, build_filter_repr_dict, description_and_settings_converter, populate_embed_from_dict ) -from bot.exts.filtering._utils import past_tense, to_serializable +from bot.exts.filtering._utils import past_tense, repr_equals, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.messages import format_channel, format_user @@ -269,7 +269,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: return filter_, filter_list, list_type = result - overrides_values, extra_fields_overrides = self._filter_overrides(filter_) + overrides_values, extra_fields_overrides = self._filter_overrides(filter_, filter_list, list_type) all_settings_repr_dict = build_filter_repr_dict( filter_list, list_type, type(filter_), overrides_values, extra_fields_overrides @@ -371,9 +371,13 @@ async def f_edit( return filter_, filter_list, list_type = result filter_type = type(filter_) - settings, filter_settings = self._filter_overrides(filter_) + settings, filter_settings = self._filter_overrides(filter_, filter_list, list_type) description, new_settings, new_filter_settings = description_and_settings_converter( - filter_list.name, self.loaded_settings, self.loaded_filter_settings, description_and_settings + filter_list, + list_type, filter_type, + self.loaded_settings, + self.loaded_filter_settings, + description_and_settings ) content = filter_.content @@ -620,15 +624,18 @@ def _get_filter_by_id(self, id_: int) -> Optional[tuple[Filter, FilterList, List return sublist[id_], filter_list, list_type @staticmethod - def _filter_overrides(filter_: Filter) -> tuple[dict, dict]: + def _filter_overrides(filter_: Filter, filter_list: FilterList, list_type: ListType) -> tuple[dict, dict]: """Get the filter's overrides to the filter list settings and the extra fields settings.""" overrides_values = {} for settings in (filter_.actions, filter_.validations): if settings: for _, setting in settings.items(): - overrides_values.update(to_serializable(setting.dict())) + for setting_name, value in to_serializable(setting.dict()).items(): + if not repr_equals(value, filter_list.default(list_type, setting_name)): + overrides_values[setting_name] = value if filter_.extra_fields_type: + # The values here can be safely used since overrides equal to the defaults won't be saved. extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) else: extra_fields_overrides = {} @@ -645,10 +652,15 @@ async def _add_filter( description_and_settings: Optional[str] = None ) -> None: """Add a filter to the database.""" + filter_type = filter_list.get_filter_type(content) description, settings, filter_settings = description_and_settings_converter( - filter_list.name, self.loaded_settings, self.loaded_filter_settings, description_and_settings + filter_list, + list_type, + filter_type, + self.loaded_settings, + self.loaded_filter_settings, + description_and_settings ) - filter_type = filter_list.get_filter_type(content) if noui: try: From 67bb6b8e36b7202c83094be529f71aab0c0d9f36 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 9 Oct 2022 00:45:15 +0300 Subject: [PATCH 041/132] Add filter template option An option is added to filter add and edit to copy the overrides over from another filter. --- bot/exts/filtering/_ui.py | 88 ++++++++++++++++++++++++++++++++- bot/exts/filtering/filtering.py | 38 +++++--------- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui.py index 8bfcded773..a6bc1addd6 100644 --- a/bot/exts/filtering/_ui.py +++ b/bot/exts/filtering/_ui.py @@ -383,6 +383,21 @@ def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Calla self.add_item(self.EnumSelect(setting_name, enum_cls, update_callback)) +class TemplateModal(discord.ui.Modal, title="Template"): + """A modal to enter a filter ID to copy its overrides over.""" + + template = discord.ui.TextInput(label="Template Filter ID") + + def __init__(self, embed_view: SettingsEditView, message: discord.Message): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.embed_view = embed_view + self.message = message + + async def on_submit(self, interaction: Interaction) -> None: + """Update the embed with the new description.""" + await self.embed_view.apply_template(self.template.value, self.message, interaction) + + class SettingsEditView(discord.ui.View): """A view used to edit a filter's settings before updating the database.""" @@ -470,6 +485,12 @@ async def empty_description(self, interaction: Interaction, button: discord.ui.B """A button to empty the filter's description.""" await self.update_embed(interaction, description=self._REMOVE) + @discord.ui.button(label="Template", row=3) + async def enter_template(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to enter a filter template ID and copy its overrides over.""" + modal = TemplateModal(self, interaction.message) + await interaction.response.send_modal(modal) + @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green, row=4) async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: """Confirm the content, description, and settings, and update the filters database.""" @@ -599,7 +620,7 @@ async def update_embed( await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) else: await interaction_or_msg.edit(embed=self.embed, view=new_view) - except discord.errors.HTTPException: # Various error such as embed description being too long. + except discord.errors.HTTPException: # Various errors such as embed description being too long. pass else: self.stop() @@ -612,6 +633,21 @@ async def edit_setting_override(self, interaction: Interaction, setting_name: st """ await self.update_embed(interaction, setting_name=setting_name, setting_value=override_value) + async def apply_template(self, template_id: str, embed_message: discord.Message, interaction: Interaction) -> None: + """Replace any non-overridden settings with overrides from the given filter.""" + try: + settings, filter_settings = template_settings(template_id, self.filter_list, self.list_type) + except ValueError as e: # The interaction is necessary to send an ephemeral message. + await interaction.response.send_message(f":x: {e}", ephemeral=True) + return + else: + await interaction.response.defer() + + self.settings_overrides = settings | self.settings_overrides + self.filter_settings_overrides = filter_settings | self.filter_settings_overrides + self.embed.clear_fields() + await embed_message.edit(embed=self.embed, view=self.copy()) + async def _remove_override(self, interaction: Interaction, select: discord.ui.Select) -> None: """ Remove the override for the setting the user selected, and edit the embed. @@ -690,6 +726,9 @@ def description_and_settings_converter( description, *parsed = parsed settings = {setting: value for setting, value in [part.split("=", maxsplit=1) for part in parsed]} + template = None + if "--template" in settings: + template = settings.pop("--template") filter_settings = {} for setting, _ in list(settings.items()): @@ -723,9 +762,56 @@ def description_and_settings_converter( except (TypeError, ValueError) as e: raise BadArgument(e) + # Pull templates settings and apply them. + if template is not None: + try: + t_settings, t_filter_settings = template_settings(template, filter_list, list_type) + except ValueError as e: + raise BadArgument(str(e)) + else: + # The specified settings go on top of the template + settings = t_settings | settings + filter_settings = t_filter_settings | filter_settings + return description, settings, filter_settings +def filter_overrides(filter_: Filter, filter_list: FilterList, list_type: ListType) -> tuple[dict, dict]: + """Get the filter's overrides to the filter list settings and the extra fields settings.""" + overrides_values = {} + for settings in (filter_.actions, filter_.validations): + if settings: + for _, setting in settings.items(): + for setting_name, value in to_serializable(setting.dict()).items(): + if not repr_equals(value, filter_list.default(list_type, setting_name)): + overrides_values[setting_name] = value + + if filter_.extra_fields_type: + # The values here can be safely used since overrides equal to the defaults won't be saved. + extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) + else: + extra_fields_overrides = {} + + return overrides_values, extra_fields_overrides + + +def template_settings(filter_id: str, filter_list: FilterList, list_type: ListType) -> tuple[dict, dict]: + """Find the filter with specified ID, and return its settings.""" + try: + filter_id = int(filter_id) + if filter_id < 0: + raise ValueError() + except ValueError: + raise ValueError("Template value must be a non-negative integer.") + + if filter_id not in filter_list.filter_lists[list_type]: + raise ValueError( + f"Could not find filter with ID `{filter_id}` in the {list_type.name} {filter_list.name} list." + ) + filter_ = filter_list.filter_lists[list_type][filter_id] + return filter_overrides(filter_, filter_list, list_type) + + def format_response_error(e: ResponseCodeError) -> Embed: """Format the response error into an embed.""" description = "" diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index aa90d1600b..427735add5 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -19,9 +19,10 @@ from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._ui import ( - ArgumentCompletionView, build_filter_repr_dict, description_and_settings_converter, populate_embed_from_dict + ArgumentCompletionView, build_filter_repr_dict, description_and_settings_converter, filter_overrides, + populate_embed_from_dict ) -from bot.exts.filtering._utils import past_tense, repr_equals, to_serializable +from bot.exts.filtering._utils import past_tense, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.messages import format_channel, format_user @@ -269,7 +270,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: return filter_, filter_list, list_type = result - overrides_values, extra_fields_overrides = self._filter_overrides(filter_, filter_list, list_type) + overrides_values, extra_fields_overrides = filter_overrides(filter_, filter_list, list_type) all_settings_repr_dict = build_filter_repr_dict( filter_list, list_type, type(filter_), overrides_values, extra_fields_overrides @@ -337,7 +338,10 @@ async def f_add( The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. - Example: `!filter add denied token "Scaleios is great" delete_messages=True send_alert=False` + A template filter can be specified in the settings area to copy overrides from. The setting name is "--template" + and the value is the filter ID. The template will be used before applying any other override. + + Example: `!filter add denied token "Scaleios is great" delete_messages=True send_alert=False --template=100` """ result = await self._resolve_list_type_and_name(ctx, list_type, list_name) if result is None: @@ -363,6 +367,9 @@ async def f_edit( The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. + A template filter can be specified in the settings area to copy overrides from. The setting name is "--template" + and the value is the filter ID. The template will be used before applying any other override. + To edit the filter's content, use the UI. """ result = self._get_filter_by_id(filter_id) @@ -371,7 +378,7 @@ async def f_edit( return filter_, filter_list, list_type = result filter_type = type(filter_) - settings, filter_settings = self._filter_overrides(filter_, filter_list, list_type) + settings, filter_settings = filter_overrides(filter_, filter_list, list_type) description, new_settings, new_filter_settings = description_and_settings_converter( filter_list, list_type, filter_type, @@ -623,25 +630,6 @@ def _get_filter_by_id(self, id_: int) -> Optional[tuple[Filter, FilterList, List if id_ in sublist: return sublist[id_], filter_list, list_type - @staticmethod - def _filter_overrides(filter_: Filter, filter_list: FilterList, list_type: ListType) -> tuple[dict, dict]: - """Get the filter's overrides to the filter list settings and the extra fields settings.""" - overrides_values = {} - for settings in (filter_.actions, filter_.validations): - if settings: - for _, setting in settings.items(): - for setting_name, value in to_serializable(setting.dict()).items(): - if not repr_equals(value, filter_list.default(list_type, setting_name)): - overrides_values[setting_name] = value - - if filter_.extra_fields_type: - # The values here can be safely used since overrides equal to the defaults won't be saved. - extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) - else: - extra_fields_overrides = {} - - return overrides_values, extra_fields_overrides - async def _add_filter( self, ctx: Context, @@ -736,7 +724,7 @@ async def _post_new_filter( "filter_list": list_id, "content": content, "description": description, "additional_field": json.dumps(filter_settings), **settings } - response = await bot.instance.api_client.post('bot/filter/filters', json=payload) + response = await bot.instance.api_client.post('bot/filter/filters', json=to_serializable(payload)) new_filter = filter_list.add_filter(response, list_type) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, new_filter) await msg.reply(f"✅ Added filter: {new_filter}" + extra_msg) From 6aeab233faec9d55d63fb53fa65df6c7d7bb0902 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 10 Oct 2022 01:16:42 +0300 Subject: [PATCH 042/132] Added filter list deletion command. --- .../filtering/_filter_lists/filter_list.py | 8 ++++ bot/exts/filtering/_ui/__init__.py | 0 bot/exts/filtering/{_ui.py => _ui/filter.py} | 0 bot/exts/filtering/_ui/filter_list.py | 31 ++++++++++++ bot/exts/filtering/filtering.py | 48 +++++++++++++++++-- 5 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 bot/exts/filtering/_ui/__init__.py rename bot/exts/filtering/{_ui.py => _ui/filter.py} (100%) create mode 100644 bot/exts/filtering/_ui/filter_list.py diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index a620131924..a4f22aed47 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -61,6 +61,14 @@ def add_list(self, list_data: dict) -> None: for filter_data in list_data["filters"]: self.add_filter(filter_data, list_type) + def remove_list(self, list_type: ListType) -> None: + """Remove the list associated with the given type from the FilterList object.""" + if list_type not in self.filter_lists: + return + self.filter_lists.pop(list_type) + self.defaults.pop(list_type) + self.list_ids.pop(list_type) + def add_filter(self, filter_data: dict, list_type: ListType) -> Filter: """Add a filter to the list of the specified type.""" try: diff --git a/bot/exts/filtering/_ui/__init__.py b/bot/exts/filtering/_ui/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bot/exts/filtering/_ui.py b/bot/exts/filtering/_ui/filter.py similarity index 100% rename from bot/exts/filtering/_ui.py rename to bot/exts/filtering/_ui/filter.py diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py new file mode 100644 index 0000000000..26852f13bd --- /dev/null +++ b/bot/exts/filtering/_ui/filter_list.py @@ -0,0 +1,31 @@ +from typing import Callable + +import discord +from discord import Interaction, Member, User + +# Amount of seconds to confirm the operation. +DELETION_TIMEOUT = 60 + + +class DeleteConfirmationView(discord.ui.View): + """A view to confirm the deletion of a filter list.""" + + def __init__(self, author: Member | User, callback: Callable): + super().__init__(timeout=DELETION_TIMEOUT) + self.author = author + self.callback = callback + + async def interaction_check(self, interaction: Interaction) -> bool: + """Only allow interactions from the command invoker.""" + return interaction.user.id == self.author.id + + @discord.ui.button(label="Delete", style=discord.ButtonStyle.red, row=0) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Invoke the filter list deletion.""" + await interaction.response.edit_message(view=None) + await self.callback() + + @discord.ui.button(label="Cancel", row=0) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the filter list deletion.""" + await interaction.response.edit_message(content="🚫 Operation canceled.", view=None) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 427735add5..2b37f1ee54 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -3,25 +3,28 @@ import re from collections import defaultdict from functools import partial, reduce +from io import BytesIO from typing import Literal, Optional, get_type_hints +import discord from discord import Colour, Embed, HTTPException, Message from discord.ext import commands from discord.ext.commands import BadArgument, Cog, Context, has_any_role from discord.utils import escape_markdown import bot -import bot.exts.filtering._ui as filters_ui +import bot.exts.filtering._ui.filter as filters_ui from bot.bot import Bot from bot.constants import Colours, MODERATION_ROLES, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings -from bot.exts.filtering._ui import ( +from bot.exts.filtering._ui.filter import ( ArgumentCompletionView, build_filter_repr_dict, description_and_settings_converter, filter_overrides, populate_embed_from_dict ) +from bot.exts.filtering._ui.filter_list import DeleteConfirmationView from bot.exts.filtering._utils import past_tense, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator @@ -92,6 +95,15 @@ def subscribe(self, filter_list: FilterList, *events: Event) -> None: if filter_list not in self._subscriptions[event]: self._subscriptions[event].append(filter_list) + def unsubscribe(self, filter_list: FilterList, *events: Event) -> None: + """Unsubscribe a filter list from the given events. If no events given, unsubscribe from every event.""" + if not events: + events = list(self._subscriptions) + + for event in events: + if filter_list in self._subscriptions.get(event, []): + self._subscriptions[event].remove(filter_list) + def collect_loaded_types(self) -> None: """ Go over the classes used in initialization and collect them to dictionaries. @@ -483,7 +495,7 @@ async def filterlist(self, ctx: Context) -> None: @filterlist.command(name="describe", aliases=("explain", "manual", "id")) async def fl_describe( - self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None + self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None ) -> None: """Show a description of the specified filter list, or a list of possible values if no values are provided.""" if not list_type and not list_name: @@ -512,6 +524,36 @@ async def fl_describe( ) await ctx.send(embed=embed) + @filterlist.command(name="delete", aliases=("remove",)) + async def fl_delete( + self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None + ) -> None: + """Remove the filter list and all of its filters from the database.""" + async def delete_list() -> None: + """The actual removal routine.""" + list_data = await bot.instance.api_client.get(f"bot/filter/filter_lists/{list_id}") + file = discord.File(BytesIO(json.dumps(list_data, indent=4).encode("utf-8")), f"{list_description}.json") + message = await ctx.send("⏳ Annihilation in progress, please hold...", file=file) + # Unload the filter list. + filter_list.remove_list(list_type) + if not filter_list.filter_lists: # There's nothing left, remove from the cog. + self.filter_lists.pop(filter_list.name) + self.unsubscribe(filter_list) + + await bot.instance.api_client.delete(f"bot/filter/filter_lists/{list_id}") + await message.edit(content=f"✅ The {list_description} list has been deleted.") + + result = await self._resolve_list_type_and_name(ctx, list_type, list_name) + if result is None: + return + list_type, filter_list = result + list_id = filter_list.list_ids[list_type] + list_description = f"{past_tense(list_type.name.lower())} {filter_list.name}" + await ctx.reply( + f"Are you sure you want to delete the {list_description} list?", + view=DeleteConfirmationView(ctx.author, delete_list) + ) + # endregion # region: helper functions From 6fedd8f6ed734e5f2a627fd0f5654c5ecbeee62e Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 10 Oct 2022 01:38:33 +0300 Subject: [PATCH 043/132] Rearrange UI into several modules --- bot/exts/filtering/_ui/filter.py | 334 +------------------------------ bot/exts/filtering/_ui/ui.py | 332 ++++++++++++++++++++++++++++++ bot/exts/filtering/filtering.py | 4 +- 3 files changed, 344 insertions(+), 326 deletions(-) create mode 100644 bot/exts/filtering/_ui/ui.py diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index a6bc1addd6..a26f5a8410 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -3,18 +3,22 @@ import re from enum import EnumMeta from functools import partial -from typing import Any, Callable, Coroutine, Optional, TypeVar, Union +from typing import Any, Callable import discord import discord.ui from botcore.site_api import ResponseCodeError from botcore.utils import scheduling from discord import Embed, Interaction, User -from discord.ext.commands import BadArgument, Context -from discord.ui.select import MISSING, SelectOption +from discord.ext.commands import BadArgument +from discord.ui.select import SelectOption from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._ui.ui import ( + BooleanSelectView, COMPONENT_TIMEOUT, CustomCallbackSelect, EnumSelectView, FreeInputModal, SequenceEditView, + format_response_error, parse_value, remove_optional +) from bot.exts.filtering._utils import repr_equals, to_serializable from bot.log import get_logger @@ -26,76 +30,8 @@ MAX_INLINE_SIZE = 50 # Number of seconds before a settings editing view timeout. EDIT_TIMEOUT = 600 -# Number of seconds before timeout of an editing component. -COMPONENT_TIMEOUT = 180 -# Max length of modal title -MAX_MODAL_TITLE_LENGTH = 45 # Max length of modal text component label MAX_MODAL_LABEL_LENGTH = 45 -# Max number of items in a select -MAX_SELECT_ITEMS = 25 -MAX_EMBED_DESCRIPTION = 4000 - -T = TypeVar('T') - - -class ArgumentCompletionSelect(discord.ui.Select): - """A select detailing the options that can be picked to assign to a missing argument.""" - - def __init__( - self, - ctx: Context, - args: list, - arg_name: str, - options: list[str], - position: int, - converter: Optional[Callable] = None - ): - super().__init__( - placeholder=f"Select a value for {arg_name!r}", - options=[discord.SelectOption(label=option) for option in options] - ) - self.ctx = ctx - self.args = args - self.position = position - self.converter = converter - - async def callback(self, interaction: discord.Interaction) -> None: - """re-invoke the context command with the completed argument value.""" - await interaction.response.defer() - value = interaction.data["values"][0] - if self.converter: - value = self.converter(value) - args = self.args.copy() # This makes the view reusable. - args.insert(self.position, value) - log.trace(f"Argument filled with the value {value}. Re-invoking command") - await self.ctx.invoke(self.ctx.command, *args) - - -class ArgumentCompletionView(discord.ui.View): - """A view used to complete a missing argument in an in invoked command.""" - - def __init__( - self, - ctx: Context, - args: list, - arg_name: str, - options: list[str], - position: int, - converter: Optional[Callable] = None - ): - super().__init__() - log.trace(f"The {arg_name} argument was designated missing in the invocation {ctx.view.buffer!r}") - self.add_item(ArgumentCompletionSelect(ctx, args, arg_name, options, position, converter)) - self.ctx = ctx - - async def interaction_check(self, interaction: discord.Interaction) -> bool: - """Check to ensure that the interacting user is the user who invoked the command.""" - if interaction.user != self.ctx.author: - embed = discord.Embed(description="Sorry, but this dropdown menu can only be used by the original author.") - await interaction.response.send_message(embed=embed, ephemeral=True) - return False - return True def build_filter_repr_dict( @@ -145,37 +81,6 @@ def populate_embed_from_dict(embed: Embed, data: dict) -> None: embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) -class CustomCallbackSelect(discord.ui.Select): - """A selection which calls the provided callback on interaction.""" - - def __init__( - self, - callback: Callable[[Interaction, discord.ui.Select], Coroutine[None]], - *, - custom_id: str = MISSING, - placeholder: str | None = None, - min_values: int = 1, - max_values: int = 1, - options: list[SelectOption] = MISSING, - disabled: bool = False, - row: int | None = None, - ): - super().__init__( - custom_id=custom_id, - placeholder=placeholder, - min_values=min_values, - max_values=max_values, - options=options, - disabled=disabled, - row=row - ) - self.custom_callback = callback - - async def callback(self, interaction: Interaction) -> Any: - """Invoke the provided callback.""" - await self.custom_callback(interaction, self) - - class EditContentModal(discord.ui.Modal, title="Edit Content"): """A modal to input a filter's content.""" @@ -208,181 +113,6 @@ async def on_submit(self, interaction: Interaction) -> None: await self.embed_view.update_embed(self.message, description=self.description.value) -class BooleanSelectView(discord.ui.View): - """A view containing an instance of BooleanSelect.""" - - class BooleanSelect(discord.ui.Select): - """Select a true or false value and send it to the supplied callback.""" - - def __init__(self, setting_name: str, update_callback: Callable): - super().__init__(options=[SelectOption(label="True"), SelectOption(label="False")]) - self.setting_name = setting_name - self.update_callback = update_callback - - async def callback(self, interaction: Interaction) -> Any: - """Respond to the interaction by sending the boolean value to the update callback.""" - await interaction.response.edit_message(content="✅ Edit confirmed", view=None) - value = self.values[0] == "True" - await self.update_callback(setting_name=self.setting_name, setting_value=value) - - def __init__(self, setting_name: str, update_callback: Callable): - super().__init__(timeout=COMPONENT_TIMEOUT) - self.add_item(self.BooleanSelect(setting_name, update_callback)) - - -class FreeInputModal(discord.ui.Modal): - """A modal to freely enter a value for a setting.""" - - def __init__(self, setting_name: str, required: bool, type_: type, update_callback: Callable): - title = f"{setting_name} Input" if len(setting_name) < MAX_MODAL_TITLE_LENGTH - 6 else "Setting Input" - super().__init__(timeout=COMPONENT_TIMEOUT, title=title) - - self.setting_name = setting_name - self.type_ = type_ - self.update_callback = update_callback - - label = setting_name if len(setting_name) < MAX_MODAL_TITLE_LENGTH else "Value" - self.setting_input = discord.ui.TextInput(label=label, style=discord.TextStyle.paragraph, required=required) - self.add_item(self.setting_input) - - async def on_submit(self, interaction: Interaction) -> None: - """Update the setting with the new value in the embed.""" - try: - value = self.type_(self.setting_input.value) - except (ValueError, TypeError): - await interaction.response.send_message( - f"Could not process the input value for `{self.setting_name}`.", ephemeral=True - ) - else: - await interaction.response.defer() - await self.update_callback(setting_name=self.setting_name, setting_value=value) - - -class SequenceEditView(discord.ui.View): - """A view to modify the contents of a sequence of values.""" - - class SingleItemModal(discord.ui.Modal): - """A modal to enter a single list item.""" - - new_item = discord.ui.TextInput(label="New Item") - - def __init__(self, view: SequenceEditView): - super().__init__(title="Item Addition", timeout=COMPONENT_TIMEOUT) - self.view = view - - async def on_submit(self, interaction: Interaction) -> None: - """Send the submitted value to be added to the list.""" - await self.view.apply_addition(interaction, self.new_item.value) - - class NewListModal(discord.ui.Modal): - """A modal to enter new contents for the list.""" - - new_value = discord.ui.TextInput(label="Enter comma separated values", style=discord.TextStyle.paragraph) - - def __init__(self, view: SequenceEditView): - super().__init__(title="New List", timeout=COMPONENT_TIMEOUT) - self.view = view - - async def on_submit(self, interaction: Interaction) -> None: - """Send the submitted value to be added to the list.""" - await self.view.apply_edit(interaction, self.new_value.value) - - def __init__(self, setting_name: str, starting_value: list, type_: type, update_callback: Callable): - super().__init__(timeout=COMPONENT_TIMEOUT) - self.setting_name = setting_name - self.stored_value = starting_value - self.type_ = type_ - self.update_callback = update_callback - - options = [SelectOption(label=item) for item in starting_value[:MAX_SELECT_ITEMS]] - self.removal_select = CustomCallbackSelect( - self.apply_removal, placeholder="Enter an item to remove", options=options, row=1 - ) - if starting_value: - self.add_item(self.removal_select) - - async def apply_removal(self, interaction: Interaction, select: discord.ui.Select) -> None: - """Remove an item from the list.""" - # The value might not be stored as a string. - _i = len(self.stored_value) - for _i, element in enumerate(self.stored_value): - if str(element) == select.values[0]: - break - if _i != len(self.stored_value): - self.stored_value.pop(_i) - - select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] - if not self.stored_value: - self.remove_item(self.removal_select) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) - - async def apply_addition(self, interaction: Interaction, item: str) -> None: - """Add an item to the list.""" - if item in self.stored_value: # Ignore duplicates - await interaction.response.defer() - return - - self.stored_value.append(item) - self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] - if len(self.stored_value) == 1: - self.add_item(self.removal_select) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) - - async def apply_edit(self, interaction: Interaction, new_list: str) -> None: - """Change the contents of the list.""" - self.stored_value = list(set(new_list.split(","))) - self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] - if len(self.stored_value) == 1: - self.add_item(self.removal_select) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) - - @discord.ui.button(label="Add Value") - async def add_value(self, interaction: Interaction, button: discord.ui.Button) -> None: - """A button to add an item to the list.""" - await interaction.response.send_modal(self.SingleItemModal(self)) - - @discord.ui.button(label="Free Input") - async def free_input(self, interaction: Interaction, button: discord.ui.Button) -> None: - """A button to change the entire list.""" - await interaction.response.send_modal(self.NewListModal(self)) - - @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green) - async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: - """Send the final value to the embed editor.""" - # Edit first, it might time out otherwise. - await interaction.response.edit_message(content="✅ Edit confirmed", view=None) - await self.update_callback(setting_name=self.setting_name, setting_value=self.stored_value) - self.stop() - - @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red) - async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: - """Cancel the list editing.""" - await interaction.response.edit_message(content="🚫 Canceled", view=None) - self.stop() - - -class EnumSelectView(discord.ui.View): - """A view containing an instance of EnumSelect.""" - - class EnumSelect(discord.ui.Select): - """Select an enum value and send it to the supplied callback.""" - - def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): - super().__init__(options=[SelectOption(label=elem.name) for elem in enum_cls]) - self.setting_name = setting_name - self.enum_cls = enum_cls - self.update_callback = update_callback - - async def callback(self, interaction: Interaction) -> Any: - """Respond to the interaction by sending the enum value to the update callback.""" - await interaction.response.edit_message(content="✅ Edit confirmed", view=None) - await self.update_callback(setting_name=self.setting_name, setting_value=self.values[0]) - - def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): - super().__init__(timeout=COMPONENT_TIMEOUT) - self.add_item(self.EnumSelect(setting_name, enum_cls, update_callback)) - - class TemplateModal(discord.ui.Modal, title="Template"): """A modal to enter a filter ID to copy its overrides over.""" @@ -533,7 +263,7 @@ async def _prompt_new_override(self, interaction: Interaction, select: discord.u """Prompt the user to give an override value for the setting they selected, and respond to the interaction.""" setting_name = select.values[0] type_ = self.type_per_setting_name[setting_name] - is_optional, type_ = _remove_optional(type_) + is_optional, type_ = remove_optional(type_) if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias type_ = type_.__origin__ new_view = self.copy() @@ -674,34 +404,6 @@ def copy(self) -> SettingsEditView: ) -def _remove_optional(type_: type) -> tuple[bool, type]: - """Return whether the type is Optional, and the Union of types which aren't None.""" - if not hasattr(type_, "__args__"): - return False, type_ - args = list(type_.__args__) - if type(None) not in args: - return False, type_ - args.remove(type(None)) - return True, Union[tuple(args)] - - -def _parse_value(value: str, type_: type[T]) -> T: - """Parse the value and attempt to convert it to the provided type.""" - is_optional, type_ = _remove_optional(type_) - if is_optional and value == '""': - return None - if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias - type_ = type_.__origin__ - if type_ in (tuple, list, set): - return type_(value.split(",")) - if type_ is bool: - return value == "True" - if isinstance(type_, EnumMeta): - return type_[value.upper()] - - return type_(value) - - def description_and_settings_converter( filter_list: FilterList, list_type: ListType, @@ -745,7 +447,7 @@ def description_and_settings_converter( raise BadArgument(f"{setting!r} is not a recognized setting.") type_ = loaded_filter_settings[filter_list.name][filter_setting_name][2] try: - parsed_value = _parse_value(settings.pop(setting), type_) + parsed_value = parse_value(settings.pop(setting), type_) if not repr_equals(parsed_value, getattr(filter_type.extra_fields_type(), filter_setting_name)): filter_settings[filter_setting_name] = parsed_value except (TypeError, ValueError) as e: @@ -756,7 +458,7 @@ def description_and_settings_converter( else: type_ = loaded_settings[setting][2] try: - parsed_value = _parse_value(settings.pop(setting), type_) + parsed_value = parse_value(settings.pop(setting), type_) if not repr_equals(parsed_value, filter_list.default(list_type, setting)): settings[setting] = parsed_value except (TypeError, ValueError) as e: @@ -810,19 +512,3 @@ def template_settings(filter_id: str, filter_list: FilterList, list_type: ListTy ) filter_ = filter_list.filter_lists[list_type][filter_id] return filter_overrides(filter_, filter_list, list_type) - - -def format_response_error(e: ResponseCodeError) -> Embed: - """Format the response error into an embed.""" - description = "" - if "non_field_errors" in e.response_json: - non_field_errors = e.response_json.pop("non_field_errors") - description += "\n".join(f"• {error}" for error in non_field_errors) + "\n" - for field, errors in e.response_json.items(): - description += "\n".join(f"• {field} - {error}" for error in errors) + "\n" - description = description.strip() - if len(description) > MAX_EMBED_DESCRIPTION: - description = description[:MAX_EMBED_DESCRIPTION] + "[...]" - - embed = Embed(colour=discord.Colour.red(), title="Oops...", description=description) - return embed diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py new file mode 100644 index 0000000000..b31094b253 --- /dev/null +++ b/bot/exts/filtering/_ui/ui.py @@ -0,0 +1,332 @@ +from __future__ import annotations + +from enum import EnumMeta +from typing import Any, Callable, Coroutine, Optional, TypeVar, Union + +import discord +from botcore.site_api import ResponseCodeError +from botcore.utils.logging import get_logger +from discord import Embed, Interaction +from discord.ext.commands import Context +from discord.ui.select import MISSING, SelectOption + +log = get_logger(__name__) + +# Number of seconds before timeout of an editing component. +COMPONENT_TIMEOUT = 180 +# Max length of modal title +MAX_MODAL_TITLE_LENGTH = 45 +# Max number of items in a select +MAX_SELECT_ITEMS = 25 +MAX_EMBED_DESCRIPTION = 4000 + +T = TypeVar('T') + + +def remove_optional(type_: type) -> tuple[bool, type]: + """Return whether the type is Optional, and the Union of types which aren't None.""" + if not hasattr(type_, "__args__"): + return False, type_ + args = list(type_.__args__) + if type(None) not in args: + return False, type_ + args.remove(type(None)) + return True, Union[tuple(args)] + + +def parse_value(value: str, type_: type[T]) -> T: + """Parse the value and attempt to convert it to the provided type.""" + is_optional, type_ = remove_optional(type_) + if is_optional and value == '""': + return None + if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias + type_ = type_.__origin__ + if type_ in (tuple, list, set): + return type_(value.split(",")) + if type_ is bool: + return value == "True" + if isinstance(type_, EnumMeta): + return type_[value.upper()] + + return type_(value) + + +def format_response_error(e: ResponseCodeError) -> Embed: + """Format the response error into an embed.""" + description = "" + if "non_field_errors" in e.response_json: + non_field_errors = e.response_json.pop("non_field_errors") + description += "\n".join(f"• {error}" for error in non_field_errors) + "\n" + for field, errors in e.response_json.items(): + description += "\n".join(f"• {field} - {error}" for error in errors) + "\n" + description = description.strip() + if len(description) > MAX_EMBED_DESCRIPTION: + description = description[:MAX_EMBED_DESCRIPTION] + "[...]" + + embed = Embed(colour=discord.Colour.red(), title="Oops...", description=description) + return embed + + +class ArgumentCompletionSelect(discord.ui.Select): + """A select detailing the options that can be picked to assign to a missing argument.""" + + def __init__( + self, + ctx: Context, + args: list, + arg_name: str, + options: list[str], + position: int, + converter: Optional[Callable] = None + ): + super().__init__( + placeholder=f"Select a value for {arg_name!r}", + options=[discord.SelectOption(label=option) for option in options] + ) + self.ctx = ctx + self.args = args + self.position = position + self.converter = converter + + async def callback(self, interaction: discord.Interaction) -> None: + """re-invoke the context command with the completed argument value.""" + await interaction.response.defer() + value = interaction.data["values"][0] + if self.converter: + value = self.converter(value) + args = self.args.copy() # This makes the view reusable. + args.insert(self.position, value) + log.trace(f"Argument filled with the value {value}. Re-invoking command") + await self.ctx.invoke(self.ctx.command, *args) + + +class ArgumentCompletionView(discord.ui.View): + """A view used to complete a missing argument in an in invoked command.""" + + def __init__( + self, + ctx: Context, + args: list, + arg_name: str, + options: list[str], + position: int, + converter: Optional[Callable] = None + ): + super().__init__() + log.trace(f"The {arg_name} argument was designated missing in the invocation {ctx.view.buffer!r}") + self.add_item(ArgumentCompletionSelect(ctx, args, arg_name, options, position, converter)) + self.ctx = ctx + + async def interaction_check(self, interaction: discord.Interaction) -> bool: + """Check to ensure that the interacting user is the user who invoked the command.""" + if interaction.user != self.ctx.author: + embed = discord.Embed(description="Sorry, but this dropdown menu can only be used by the original author.") + await interaction.response.send_message(embed=embed, ephemeral=True) + return False + return True + + +class CustomCallbackSelect(discord.ui.Select): + """A selection which calls the provided callback on interaction.""" + + def __init__( + self, + callback: Callable[[Interaction, discord.ui.Select], Coroutine[None]], + *, + custom_id: str = MISSING, + placeholder: str | None = None, + min_values: int = 1, + max_values: int = 1, + options: list[SelectOption] = MISSING, + disabled: bool = False, + row: int | None = None, + ): + super().__init__( + custom_id=custom_id, + placeholder=placeholder, + min_values=min_values, + max_values=max_values, + options=options, + disabled=disabled, + row=row + ) + self.custom_callback = callback + + async def callback(self, interaction: Interaction) -> Any: + """Invoke the provided callback.""" + await self.custom_callback(interaction, self) + + +class BooleanSelectView(discord.ui.View): + """A view containing an instance of BooleanSelect.""" + + class BooleanSelect(discord.ui.Select): + """Select a true or false value and send it to the supplied callback.""" + + def __init__(self, setting_name: str, update_callback: Callable): + super().__init__(options=[SelectOption(label="True"), SelectOption(label="False")]) + self.setting_name = setting_name + self.update_callback = update_callback + + async def callback(self, interaction: Interaction) -> Any: + """Respond to the interaction by sending the boolean value to the update callback.""" + await interaction.response.edit_message(content="✅ Edit confirmed", view=None) + value = self.values[0] == "True" + await self.update_callback(setting_name=self.setting_name, setting_value=value) + + def __init__(self, setting_name: str, update_callback: Callable): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.add_item(self.BooleanSelect(setting_name, update_callback)) + + +class FreeInputModal(discord.ui.Modal): + """A modal to freely enter a value for a setting.""" + + def __init__(self, setting_name: str, required: bool, type_: type, update_callback: Callable): + title = f"{setting_name} Input" if len(setting_name) < MAX_MODAL_TITLE_LENGTH - 6 else "Setting Input" + super().__init__(timeout=COMPONENT_TIMEOUT, title=title) + + self.setting_name = setting_name + self.type_ = type_ + self.update_callback = update_callback + + label = setting_name if len(setting_name) < MAX_MODAL_TITLE_LENGTH else "Value" + self.setting_input = discord.ui.TextInput(label=label, style=discord.TextStyle.paragraph, required=required) + self.add_item(self.setting_input) + + async def on_submit(self, interaction: Interaction) -> None: + """Update the setting with the new value in the embed.""" + try: + value = self.type_(self.setting_input.value) + except (ValueError, TypeError): + await interaction.response.send_message( + f"Could not process the input value for `{self.setting_name}`.", ephemeral=True + ) + else: + await interaction.response.defer() + await self.update_callback(setting_name=self.setting_name, setting_value=value) + + +class SequenceEditView(discord.ui.View): + """A view to modify the contents of a sequence of values.""" + + class SingleItemModal(discord.ui.Modal): + """A modal to enter a single list item.""" + + new_item = discord.ui.TextInput(label="New Item") + + def __init__(self, view: SequenceEditView): + super().__init__(title="Item Addition", timeout=COMPONENT_TIMEOUT) + self.view = view + + async def on_submit(self, interaction: Interaction) -> None: + """Send the submitted value to be added to the list.""" + await self.view.apply_addition(interaction, self.new_item.value) + + class NewListModal(discord.ui.Modal): + """A modal to enter new contents for the list.""" + + new_value = discord.ui.TextInput(label="Enter comma separated values", style=discord.TextStyle.paragraph) + + def __init__(self, view: SequenceEditView): + super().__init__(title="New List", timeout=COMPONENT_TIMEOUT) + self.view = view + + async def on_submit(self, interaction: Interaction) -> None: + """Send the submitted value to be added to the list.""" + await self.view.apply_edit(interaction, self.new_value.value) + + def __init__(self, setting_name: str, starting_value: list, type_: type, update_callback: Callable): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.setting_name = setting_name + self.stored_value = starting_value + self.type_ = type_ + self.update_callback = update_callback + + options = [SelectOption(label=item) for item in starting_value[:MAX_SELECT_ITEMS]] + self.removal_select = CustomCallbackSelect( + self.apply_removal, placeholder="Enter an item to remove", options=options, row=1 + ) + if starting_value: + self.add_item(self.removal_select) + + async def apply_removal(self, interaction: Interaction, select: discord.ui.Select) -> None: + """Remove an item from the list.""" + # The value might not be stored as a string. + _i = len(self.stored_value) + for _i, element in enumerate(self.stored_value): + if str(element) == select.values[0]: + break + if _i != len(self.stored_value): + self.stored_value.pop(_i) + + select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] + if not self.stored_value: + self.remove_item(self.removal_select) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + + async def apply_addition(self, interaction: Interaction, item: str) -> None: + """Add an item to the list.""" + if item in self.stored_value: # Ignore duplicates + await interaction.response.defer() + return + + self.stored_value.append(item) + self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] + if len(self.stored_value) == 1: + self.add_item(self.removal_select) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + + async def apply_edit(self, interaction: Interaction, new_list: str) -> None: + """Change the contents of the list.""" + self.stored_value = list(set(new_list.split(","))) + self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] + if len(self.stored_value) == 1: + self.add_item(self.removal_select) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + + @discord.ui.button(label="Add Value") + async def add_value(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to add an item to the list.""" + await interaction.response.send_modal(self.SingleItemModal(self)) + + @discord.ui.button(label="Free Input") + async def free_input(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to change the entire list.""" + await interaction.response.send_modal(self.NewListModal(self)) + + @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Send the final value to the embed editor.""" + # Edit first, it might time out otherwise. + await interaction.response.edit_message(content="✅ Edit confirmed", view=None) + await self.update_callback(setting_name=self.setting_name, setting_value=self.stored_value) + self.stop() + + @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the list editing.""" + await interaction.response.edit_message(content="🚫 Canceled", view=None) + self.stop() + + +class EnumSelectView(discord.ui.View): + """A view containing an instance of EnumSelect.""" + + class EnumSelect(discord.ui.Select): + """Select an enum value and send it to the supplied callback.""" + + def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): + super().__init__(options=[SelectOption(label=elem.name) for elem in enum_cls]) + self.setting_name = setting_name + self.enum_cls = enum_cls + self.update_callback = update_callback + + async def callback(self, interaction: Interaction) -> Any: + """Respond to the interaction by sending the enum value to the update callback.""" + await interaction.response.edit_message(content="✅ Edit confirmed", view=None) + await self.update_callback(setting_name=self.setting_name, setting_value=self.values[0]) + + def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.add_item(self.EnumSelect(setting_name, enum_cls, update_callback)) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 2b37f1ee54..0bcf485c00 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -21,10 +21,10 @@ from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._ui.filter import ( - ArgumentCompletionView, build_filter_repr_dict, description_and_settings_converter, filter_overrides, - populate_embed_from_dict + build_filter_repr_dict, description_and_settings_converter, filter_overrides, populate_embed_from_dict ) from bot.exts.filtering._ui.filter_list import DeleteConfirmationView +from bot.exts.filtering._ui.ui import ArgumentCompletionView from bot.exts.filtering._utils import past_tense, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator From 02fedb2304b85c734dce8efc08d54106acdc8986 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 10 Oct 2022 22:40:05 +0300 Subject: [PATCH 044/132] Add filter list edit command The UI is a simplified version of the filter UI. In fact the two views now use the same base class. Also fixes a bug in filters with displaying the correct value in the embed for filter settings. --- bot/exts/filtering/_filter_lists/domain.py | 2 +- bot/exts/filtering/_ui/filter.py | 102 +++---------- bot/exts/filtering/_ui/filter_list.py | 166 +++++++++++++++++++- bot/exts/filtering/_ui/ui.py | 97 +++++++++++- bot/exts/filtering/filtering.py | 170 +++++++++++++-------- 5 files changed, 393 insertions(+), 144 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index c407108caa..cae2eb8782 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -27,7 +27,7 @@ class DomainsList(FilterList): individual filters. Domains are found by looking for a URL schema (http or https). - Filters will also trigger for subdomains unless set otherwise. + Filters will also trigger for subdomains. """ name = "domain" diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index a26f5a8410..4da8fe0019 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -1,14 +1,10 @@ from __future__ import annotations -import re -from enum import EnumMeta -from functools import partial from typing import Any, Callable import discord import discord.ui from botcore.site_api import ResponseCodeError -from botcore.utils import scheduling from discord import Embed, Interaction, User from discord.ext.commands import BadArgument from discord.ui.select import SelectOption @@ -16,23 +12,14 @@ from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._ui.ui import ( - BooleanSelectView, COMPONENT_TIMEOUT, CustomCallbackSelect, EnumSelectView, FreeInputModal, SequenceEditView, - format_response_error, parse_value, remove_optional + COMPONENT_TIMEOUT, CustomCallbackSelect, EditBaseView, MISSING, SETTINGS_DELIMITER, SINGLE_SETTING_PATTERN, + format_response_error, parse_value, populate_embed_from_dict ) from bot.exts.filtering._utils import repr_equals, to_serializable from bot.log import get_logger log = get_logger(__name__) -# Max number of characters in a Discord embed field value, minus 6 characters for a placeholder. -MAX_FIELD_SIZE = 1018 -# Max number of characters for an embed field's value before it should take its own line. -MAX_INLINE_SIZE = 50 -# Number of seconds before a settings editing view timeout. -EDIT_TIMEOUT = 600 -# Max length of modal text component label -MAX_MODAL_LABEL_LENGTH = 45 - def build_filter_repr_dict( filter_list: FilterList, @@ -63,30 +50,17 @@ def build_filter_repr_dict( if name not in extra_fields_overrides or repr_equals(extra_fields_overrides[name], value): total_values[f"{filter_type.name}/{name}"] = value else: - total_values[f"{filter_type.name}/{name}*"] = value + total_values[f"{filter_type.name}/{name}*"] = extra_fields_overrides[name] return total_values -def populate_embed_from_dict(embed: Embed, data: dict) -> None: - """Populate a Discord embed by populating fields from the given dict.""" - for setting, value in data.items(): - if setting.startswith("_"): - continue - if type(value) in (set, tuple): - value = list(value) - value = str(value) if value not in ("", None) else "-" - if len(value) > MAX_FIELD_SIZE: - value = value[:MAX_FIELD_SIZE] + " [...]" - embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) - - class EditContentModal(discord.ui.Modal, title="Edit Content"): """A modal to input a filter's content.""" content = discord.ui.TextInput(label="Content") - def __init__(self, embed_view: SettingsEditView, message: discord.Message): + def __init__(self, embed_view: FilterEditView, message: discord.Message): super().__init__(timeout=COMPONENT_TIMEOUT) self.embed_view = embed_view self.message = message @@ -102,7 +76,7 @@ class EditDescriptionModal(discord.ui.Modal, title="Edit Description"): description = discord.ui.TextInput(label="Description") - def __init__(self, embed_view: SettingsEditView, message: discord.Message): + def __init__(self, embed_view: FilterEditView, message: discord.Message): super().__init__(timeout=COMPONENT_TIMEOUT) self.embed_view = embed_view self.message = message @@ -118,7 +92,7 @@ class TemplateModal(discord.ui.Modal, title="Template"): template = discord.ui.TextInput(label="Template Filter ID") - def __init__(self, embed_view: SettingsEditView, message: discord.Message): + def __init__(self, embed_view: FilterEditView, message: discord.Message): super().__init__(timeout=COMPONENT_TIMEOUT) self.embed_view = embed_view self.message = message @@ -128,7 +102,7 @@ async def on_submit(self, interaction: Interaction) -> None: await self.embed_view.apply_template(self.template.value, self.message, interaction) -class SettingsEditView(discord.ui.View): +class FilterEditView(EditBaseView): """A view used to edit a filter's settings before updating the database.""" class _REMOVE: @@ -149,7 +123,7 @@ def __init__( embed: Embed, confirm_callback: Callable ): - super().__init__(timeout=EDIT_TIMEOUT) + super().__init__(author) self.filter_list = filter_list self.list_type = list_type self.filter_type = filter_type @@ -159,7 +133,6 @@ def __init__( self.filter_settings_overrides = filter_settings_overrides self.loaded_settings = loaded_settings self.loaded_filter_settings = loaded_filter_settings - self.author = author self.embed = embed self.confirm_callback = confirm_callback @@ -175,7 +148,7 @@ def __init__( }) add_select = CustomCallbackSelect( - self._prompt_new_override, + self._prompt_new_value, placeholder="Select a setting to edit", options=[SelectOption(label=name) for name in sorted(self.type_per_setting_name)], row=1 @@ -194,10 +167,6 @@ def __init__( if remove_select.options: self.add_item(remove_select) - async def interaction_check(self, interaction: Interaction) -> bool: - """Only allow interactions from the command invoker.""" - return interaction.user.id == self.author.id - @discord.ui.button(label="Edit Content", row=3) async def edit_content(self, interaction: Interaction, button: discord.ui.Button) -> None: """A button to edit the filter's content. Pressing the button invokes a modal.""" @@ -259,44 +228,24 @@ async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> N await interaction.response.edit_message(content="🚫 Operation canceled.", embed=None, view=None) self.stop() - async def _prompt_new_override(self, interaction: Interaction, select: discord.ui.Select) -> None: - """Prompt the user to give an override value for the setting they selected, and respond to the interaction.""" - setting_name = select.values[0] - type_ = self.type_per_setting_name[setting_name] - is_optional, type_ = remove_optional(type_) - if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias - type_ = type_.__origin__ - new_view = self.copy() - # This is in order to not block the interaction response. There's a potential race condition here, since - # a view's method is used without guaranteeing the task completed, but since it depends on user input - # realistically it shouldn't happen. - scheduling.create_task(interaction.message.edit(view=new_view)) - update_callback = partial(new_view.update_embed, interaction_or_msg=interaction.message) - if type_ is bool: - view = BooleanSelectView(setting_name, update_callback) - await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) - elif type_ in (set, list, tuple): - current_value = self.settings_overrides.get(setting_name, []) - await interaction.response.send_message( - f"Current list: {current_value}", - view=SequenceEditView(setting_name, current_value, type_, update_callback), - ephemeral=True - ) - elif isinstance(type_, EnumMeta): - view = EnumSelectView(setting_name, type_, update_callback) - await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) - else: - await interaction.response.send_modal(FreeInputModal(setting_name, not is_optional, type_, update_callback)) - self.stop() + def current_value(self, setting_name: str) -> Any: + """Get the current value stored for the setting or MISSING if none found.""" + if setting_name in self.settings_overrides: + return self.settings_overrides[setting_name] + if "/" in setting_name: + _, setting_name = setting_name.split("/", maxsplit=1) + if setting_name in self.filter_settings_overrides: + return self.filter_settings_overrides[setting_name] + return MISSING async def update_embed( self, interaction_or_msg: discord.Interaction | discord.Message, *, content: str | None = None, - description: str | type[SettingsEditView._REMOVE] | None = None, + description: str | type[FilterEditView._REMOVE] | None = None, setting_name: str | None = None, - setting_value: str | type[SettingsEditView._REMOVE] | None = None, + setting_value: str | type[FilterEditView._REMOVE] | None = None, ) -> None: """ Update the embed with the new information. @@ -386,9 +335,9 @@ async def _remove_override(self, interaction: Interaction, select: discord.ui.Se """ await self.update_embed(interaction, setting_name=select.values[0], setting_value=self._REMOVE) - def copy(self) -> SettingsEditView: + def copy(self) -> FilterEditView: """Create a copy of this view.""" - return SettingsEditView( + return FilterEditView( self.filter_list, self.list_type, self.filter_type, @@ -416,15 +365,12 @@ def description_and_settings_converter( if not input_data: return "", {}, {} - settings_pattern = re.compile(r"\s+(?=\S+=\S+)") - single_setting_pattern = re.compile(r"\w+=.+") - - parsed = settings_pattern.split(input_data) + parsed = SETTINGS_DELIMITER.split(input_data) if not parsed: return "", {}, {} description = "" - if not single_setting_pattern.match(parsed[0]): + if not SINGLE_SETTING_PATTERN.match(parsed[0]): description, *parsed = parsed settings = {setting: value for setting, value in [part.split("=", maxsplit=1) for part in parsed]} diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index 26852f13bd..051521f1e5 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -1,12 +1,174 @@ -from typing import Callable +from __future__ import annotations + +from typing import Any, Callable import discord -from discord import Interaction, Member, User +from botcore.site_api import ResponseCodeError +from discord import Embed, Interaction, Member, SelectOption, User +from discord.ext.commands import BadArgument + +from bot.exts.filtering._filter_lists import FilterList, ListType +from bot.exts.filtering._ui.ui import ( + CustomCallbackSelect, EditBaseView, MISSING, SETTINGS_DELIMITER, format_response_error, parse_value, + populate_embed_from_dict +) +from bot.exts.filtering._utils import repr_equals, to_serializable # Amount of seconds to confirm the operation. DELETION_TIMEOUT = 60 +def settings_converter(loaded_settings: dict, input_data: str) -> dict[str, Any]: + """Parse a string representing settings, and validate the setting names.""" + if not input_data: + return {} + + parsed = SETTINGS_DELIMITER.split(input_data) + if not parsed: + return {} + + settings = {setting: value for setting, value in [part.split("=", maxsplit=1) for part in parsed]} + for setting in settings: + if setting not in loaded_settings: + raise BadArgument(f"{setting!r} is not a recognized setting.") + else: + type_ = loaded_settings[setting][2] + try: + parsed_value = parse_value(settings.pop(setting), type_) + settings[setting] = parsed_value + except (TypeError, ValueError) as e: + raise BadArgument(e) + + return settings + + +def build_filterlist_repr_dict(filter_list: FilterList, list_type: ListType, new_settings: dict) -> dict: + """Build a dictionary of field names and values to pass to `_build_embed_from_dict`.""" + # Get filter list settings + default_setting_values = {} + for type_ in ("actions", "validations"): + for _, setting in filter_list.defaults[list_type][type_].items(): + default_setting_values.update(to_serializable(setting.dict())) + + # Add new values. It's done in this way to preserve field order, since the new_values won't have all settings. + total_values = {} + for name, value in default_setting_values.items(): + if name not in new_settings or repr_equals(new_settings[name], value): + total_values[name] = value + else: + total_values[f"{name}~"] = new_settings[name] + + return total_values + + +class FilterListEditView(EditBaseView): + """A view used to edit a filter's settings before updating the database.""" + + def __init__( + self, + filter_list: FilterList, + list_type: ListType, + new_settings: dict, + loaded_settings: dict, + author: User, + embed: Embed, + confirm_callback: Callable + ): + super().__init__(author) + self.filter_list = filter_list + self.list_type = list_type + self.settings = new_settings + self.loaded_settings = loaded_settings + self.embed = embed + self.confirm_callback = confirm_callback + + self.settings_repr_dict = build_filterlist_repr_dict(filter_list, list_type, new_settings) + populate_embed_from_dict(embed, self.settings_repr_dict) + + self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} + + edit_select = CustomCallbackSelect( + self._prompt_new_value, + placeholder="Select a setting to edit", + options=[SelectOption(label=name) for name in sorted(self.type_per_setting_name)], + row=0 + ) + self.add_item(edit_select) + + @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green, row=1) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Confirm the content, description, and settings, and update the filters database.""" + await interaction.response.edit_message(view=None) # Make sure the interaction succeeds first. + try: + await self.confirm_callback(interaction.message, self.filter_list, self.list_type, self.settings) + except ResponseCodeError as e: + await interaction.message.reply(embed=format_response_error(e)) + await interaction.message.edit(view=self) + else: + self.stop() + + @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red, row=1) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the operation.""" + await interaction.response.edit_message(content="🚫 Operation canceled.", embed=None, view=None) + self.stop() + + def current_value(self, setting_name: str) -> Any: + """Get the current value stored for the setting or MISSING if none found.""" + if setting_name in self.settings: + return self.settings[setting_name] + if setting_name in self.settings_repr_dict: + return self.settings_repr_dict[setting_name] + return MISSING + + async def update_embed( + self, + interaction_or_msg: discord.Interaction | discord.Message, + *, + setting_name: str | None = None, + setting_value: str | None = None, + ) -> None: + """ + Update the embed with the new information. + + If `interaction_or_msg` is a Message, the invoking Interaction must be deferred before calling this function. + """ + if not setting_name: # Obligatory check to match the signature in the parent class. + return + + default_value = self.filter_list.default(self.list_type, setting_name) + if not repr_equals(setting_value, default_value): + self.settings[setting_name] = setting_value + # If there's already a new value, remove it, since the new value is the same as the default. + elif setting_name in self.settings: + self.settings.pop(setting_name) + + self.embed.clear_fields() + new_view = self.copy() + + try: + if isinstance(interaction_or_msg, discord.Interaction): + await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) + else: + await interaction_or_msg.edit(embed=self.embed, view=new_view) + except discord.errors.HTTPException: # Various errors such as embed description being too long. + pass + else: + self.stop() + + def copy(self) -> FilterListEditView: + """Create a copy of this view.""" + return FilterListEditView( + self.filter_list, + self.list_type, + self.settings, + self.loaded_settings, + self.author, + self.embed, + self.confirm_callback + ) + + class DeleteConfirmationView(discord.ui.View): """A view to confirm the deletion of a filter list.""" diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index b31094b253..dc3bd01c9f 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -1,17 +1,28 @@ from __future__ import annotations +import re +from abc import ABC, abstractmethod from enum import EnumMeta +from functools import partial from typing import Any, Callable, Coroutine, Optional, TypeVar, Union import discord from botcore.site_api import ResponseCodeError +from botcore.utils import scheduling from botcore.utils.logging import get_logger from discord import Embed, Interaction from discord.ext.commands import Context -from discord.ui.select import MISSING, SelectOption +from discord.ui.select import MISSING as SELECT_MISSING, SelectOption log = get_logger(__name__) + +# Max number of characters in a Discord embed field value, minus 6 characters for a placeholder. +MAX_FIELD_SIZE = 1018 +# Max number of characters for an embed field's value before it should take its own line. +MAX_INLINE_SIZE = 50 +# Number of seconds before a settings editing view timeout. +EDIT_TIMEOUT = 600 # Number of seconds before timeout of an editing component. COMPONENT_TIMEOUT = 180 # Max length of modal title @@ -20,9 +31,28 @@ MAX_SELECT_ITEMS = 25 MAX_EMBED_DESCRIPTION = 4000 +SETTINGS_DELIMITER = re.compile(r"\s+(?=\S+=\S+)") +SINGLE_SETTING_PATTERN = re.compile(r"\w+=.+") + +# Sentinel value to denote that a value is missing +MISSING = object() + T = TypeVar('T') +def populate_embed_from_dict(embed: Embed, data: dict) -> None: + """Populate a Discord embed by populating fields from the given dict.""" + for setting, value in data.items(): + if setting.startswith("_"): + continue + if type(value) in (set, tuple): + value = list(value) + value = str(value) if value not in ("", None) else "-" + if len(value) > MAX_FIELD_SIZE: + value = value[:MAX_FIELD_SIZE] + " [...]" + embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) + + def remove_optional(type_: type) -> tuple[bool, type]: """Return whether the type is Optional, and the Union of types which aren't None.""" if not hasattr(type_, "__args__"): @@ -133,11 +163,11 @@ def __init__( self, callback: Callable[[Interaction, discord.ui.Select], Coroutine[None]], *, - custom_id: str = MISSING, + custom_id: str = SELECT_MISSING, placeholder: str | None = None, min_values: int = 1, max_values: int = 1, - options: list[SelectOption] = MISSING, + options: list[SelectOption] = SELECT_MISSING, disabled: bool = False, row: int | None = None, ): @@ -330,3 +360,64 @@ async def callback(self, interaction: Interaction) -> Any: def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): super().__init__(timeout=COMPONENT_TIMEOUT) self.add_item(self.EnumSelect(setting_name, enum_cls, update_callback)) + + +class EditBaseView(ABC, discord.ui.View): + """A view used to edit embed fields based on a provided type.""" + + def __init__(self, author: discord.User): + super().__init__(timeout=EDIT_TIMEOUT) + self.author = author + self.type_per_setting_name = {} + + async def interaction_check(self, interaction: Interaction) -> bool: + """Only allow interactions from the command invoker.""" + return interaction.user.id == self.author.id + + async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.Select) -> None: + """Prompt the user to give an override value for the setting they selected, and respond to the interaction.""" + setting_name = select.values[0] + type_ = self.type_per_setting_name[setting_name] + is_optional, type_ = remove_optional(type_) + if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias + type_ = type_.__origin__ + new_view = self.copy() + # This is in order to not block the interaction response. There's a potential race condition here, since + # a view's method is used without guaranteeing the task completed, but since it depends on user input + # realistically it shouldn't happen. + scheduling.create_task(interaction.message.edit(view=new_view)) + update_callback = partial(new_view.update_embed, interaction_or_msg=interaction.message) + if type_ is bool: + view = BooleanSelectView(setting_name, update_callback) + await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) + elif type_ in (set, list, tuple): + current_list = self.current_value(setting_name) + if current_list is MISSING: + current_list = [] + await interaction.response.send_message( + f"Current list: {current_list}", + view=SequenceEditView(setting_name, current_list, type_, update_callback), + ephemeral=True + ) + elif isinstance(type_, EnumMeta): + view = EnumSelectView(setting_name, type_, update_callback) + await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) + else: + await interaction.response.send_modal(FreeInputModal(setting_name, not is_optional, type_, update_callback)) + self.stop() + + @abstractmethod + def current_value(self, setting_name: str) -> Any: + """Get the current value stored for the setting or MISSING if none found.""" + + @abstractmethod + async def update_embed(self, interaction_or_msg: Interaction | discord.Message) -> None: + """ + Update the embed with the new information. + + If `interaction_or_msg` is a Message, the invoking Interaction must be deferred before calling this function. + """ + + @abstractmethod + def copy(self) -> EditBaseView: + """Create a copy of this view.""" diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 0bcf485c00..a906b6a413 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -15,7 +15,7 @@ import bot import bot.exts.filtering._ui.filter as filters_ui from bot.bot import Bot -from bot.constants import Colours, MODERATION_ROLES, Webhooks +from bot.constants import Colours, MODERATION_ROLES, Roles, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filters.filter import Filter @@ -23,7 +23,7 @@ from bot.exts.filtering._ui.filter import ( build_filter_repr_dict, description_and_settings_converter, filter_overrides, populate_embed_from_dict ) -from bot.exts.filtering._ui.filter_list import DeleteConfirmationView +from bot.exts.filtering._ui.filter_list import DeleteConfirmationView, FilterListEditView, settings_converter from bot.exts.filtering._ui.ui import ArgumentCompletionView from bot.exts.filtering._utils import past_tense, to_serializable from bot.log import get_logger @@ -409,34 +409,34 @@ async def f_edit( await patch_func( ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings ) + return - else: - embed = Embed(colour=Colour.blue()) - embed.description = f"`{filter_.content}`" - if description: - embed.description += f" - {description}" - embed.set_author( - name=f"Filter #{filter_id} - {past_tense(list_type.name.lower())} {filter_list.name}".title()) - embed.set_footer(text=( - "Field names with an asterisk have values which override the defaults of the containing filter list. " - f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." - )) - - view = filters_ui.SettingsEditView( - filter_list, - list_type, - filter_type, - content, - description, - settings, - filter_settings, - self.loaded_settings, - self.loaded_filter_settings, - ctx.author, - embed, - patch_func - ) - await ctx.send(embed=embed, reference=ctx.message, view=view) + embed = Embed(colour=Colour.blue()) + embed.description = f"`{filter_.content}`" + if description: + embed.description += f" - {description}" + embed.set_author( + name=f"Filter #{filter_id} - {past_tense(list_type.name.lower())} {filter_list.name}".title()) + embed.set_footer(text=( + "Field names with an asterisk have values which override the defaults of the containing filter list. " + f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + )) + + view = filters_ui.FilterEditView( + filter_list, + list_type, + filter_type, + content, + description, + settings, + filter_settings, + self.loaded_settings, + self.loaded_filter_settings, + ctx.author, + embed, + patch_func + ) + await ctx.send(embed=embed, reference=ctx.message, view=view) @filter.command(name="delete", aliases=("d", "remove")) async def f_delete(self, ctx: Context, filter_id: int) -> None: @@ -524,7 +524,50 @@ async def fl_describe( ) await ctx.send(embed=embed) + @filterlist.command(name="edit", aliases=("e",)) + @has_any_role(Roles.admins) + async def fl_edit( + self, + ctx: Context, + noui: Optional[Literal["noui"]], + list_type: Optional[list_type_converter] = None, + list_name: Optional[str] = None, + *, + settings: str | None + ) -> None: + """ + Edit the filter list. + + Unless `noui` is specified, a UI will be provided to edit the settings before confirmation. + + The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the + equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. + """ + result = await self._resolve_list_type_and_name(ctx, list_type, list_name) + if result is None: + return + list_type, filter_list = result + settings = settings_converter(self.loaded_settings, settings) + if noui: + await self._patch_filter_list(ctx.message, filter_list, list_type, settings) + + embed = Embed(colour=Colour.blue()) + embed.set_author(name=f"{past_tense(list_type.name.lower())} {filter_list.name} Filter List".title()) + embed.set_footer(text="Field names with a ~ have values which change the existing value in the filter list.") + + view = FilterListEditView( + filter_list, + list_type, + settings, + self.loaded_settings, + ctx.author, + embed, + self._patch_filter_list + ) + await ctx.send(embed=embed, reference=ctx.message, view=view) + @filterlist.command(name="delete", aliases=("remove",)) + @has_any_role(Roles.admins) async def fl_delete( self, ctx: Context, list_type: Optional[list_type_converter] = None, list_name: Optional[str] = None ) -> None: @@ -699,34 +742,34 @@ async def _add_filter( ) except ValueError as e: raise BadArgument(str(e)) + return - else: - embed = Embed(colour=Colour.blue()) - embed.description = f"`{content}`" if content else "*No content*" - if description: - embed.description += f" - {description}" - embed.set_author( - name=f"New Filter - {past_tense(list_type.name.lower())} {filter_list.name}".title()) - embed.set_footer(text=( - "Field names with an asterisk have values which override the defaults of the containing filter list. " - f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." - )) - - view = filters_ui.SettingsEditView( - filter_list, - list_type, - filter_type, - content, - description, - settings, - filter_settings, - self.loaded_settings, - self.loaded_filter_settings, - ctx.author, - embed, - self._post_new_filter - ) - await ctx.send(embed=embed, reference=ctx.message, view=view) + embed = Embed(colour=Colour.blue()) + embed.description = f"`{content}`" if content else "*No content*" + if description: + embed.description += f" - {description}" + embed.set_author( + name=f"New Filter - {past_tense(list_type.name.lower())} {filter_list.name}".title()) + embed.set_footer(text=( + "Field names with an asterisk have values which override the defaults of the containing filter list. " + f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + )) + + view = filters_ui.FilterEditView( + filter_list, + list_type, + filter_type, + content, + description, + settings, + filter_settings, + self.loaded_settings, + self.loaded_filter_settings, + ctx.author, + embed, + self._post_new_filter + ) + await ctx.send(embed=embed, reference=ctx.message, view=view) @staticmethod def _identical_filters_message(content: str, filter_list: FilterList, list_type: ListType, filter_: Filter) -> str: @@ -794,20 +837,27 @@ async def _patch_filter( # If the setting is not in `settings`, the override was either removed, or there wasn't one in the first place. for current_settings in (filter_.actions, filter_.validations): if current_settings: - for _, setting_entry in current_settings.items(): + for setting_entry in current_settings.values(): settings.update({setting: None for setting in setting_entry.dict() if setting not in settings}) - list_id = filter_list.list_ids[list_type] description = description or None payload = { - "filter_list": list_id, "content": content, "description": description, - "additional_field": json.dumps(filter_settings), **settings + "content": content, "description": description, "additional_field": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.patch(f'bot/filter/filters/{filter_.id}', json=payload) edited_filter = filter_list.add_filter(response, list_type) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, edited_filter) await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) + @staticmethod + async def _patch_filter_list(msg: Message, filter_list: FilterList, list_type: ListType, settings: dict) -> None: + """PATCH the new data of the filter list to the site API.""" + list_id = filter_list.list_ids[list_type] + response = await bot.instance.api_client.patch(f'bot/filter/filter_lists/{list_id}', json=settings) + filter_list.remove_list(list_type) + filter_list.add_list(response) + await msg.reply(f"✅ Edited filter list: {past_tense(list_type.name.lower())} {filter_list.name}") + # endregion From fb4117bcee3028f3679ecfd99399f3b089c2beb9 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 11 Oct 2022 10:29:57 +0300 Subject: [PATCH 045/132] Filter list add command --- bot/exts/filtering/_ui/filter_list.py | 103 +++++++++++++++++++++++++- bot/exts/filtering/_ui/ui.py | 11 ++- bot/exts/filtering/_utils.py | 21 +++++- bot/exts/filtering/filtering.py | 79 +++++++++++++++----- 4 files changed, 189 insertions(+), 25 deletions(-) diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index 051521f1e5..cc58f5a62e 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -61,8 +61,109 @@ def build_filterlist_repr_dict(filter_list: FilterList, list_type: ListType, new return total_values +class FilterListAddView(EditBaseView): + """A view used to add a new filter list.""" + + def __init__( + self, + list_name: str, + list_type: ListType, + settings: dict, + loaded_settings: dict, + author: User, + embed: Embed, + confirm_callback: Callable + ): + super().__init__(author) + self.list_name = list_name + self.list_type = list_type + self.settings = settings + self.loaded_settings = loaded_settings + self.embed = embed + self.confirm_callback = confirm_callback + + self.settings_repr_dict = {name: to_serializable(value) for name, value in settings.items()} + populate_embed_from_dict(embed, self.settings_repr_dict) + + self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} + + edit_select = CustomCallbackSelect( + self._prompt_new_value, + placeholder="Select a setting to edit", + options=[SelectOption(label=name) for name in sorted(settings)], + row=0 + ) + self.add_item(edit_select) + + @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green, row=1) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Confirm the content, description, and settings, and update the filters database.""" + await interaction.response.edit_message(view=None) # Make sure the interaction succeeds first. + try: + await self.confirm_callback(interaction.message, self.list_name, self.list_type, self.settings) + except ResponseCodeError as e: + await interaction.message.reply(embed=format_response_error(e)) + await interaction.message.edit(view=self) + else: + self.stop() + + @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red, row=1) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the operation.""" + await interaction.response.edit_message(content="🚫 Operation canceled.", embed=None, view=None) + self.stop() + + def current_value(self, setting_name: str) -> Any: + """Get the current value stored for the setting or MISSING if none found.""" + if setting_name in self.settings: + return self.settings[setting_name] + return MISSING + + async def update_embed( + self, + interaction_or_msg: discord.Interaction | discord.Message, + *, + setting_name: str | None = None, + setting_value: str | None = None, + ) -> None: + """ + Update the embed with the new information. + + If `interaction_or_msg` is a Message, the invoking Interaction must be deferred before calling this function. + """ + if not setting_name: # Obligatory check to match the signature in the parent class. + return + + self.settings[setting_name] = setting_value + + self.embed.clear_fields() + new_view = self.copy() + + try: + if isinstance(interaction_or_msg, discord.Interaction): + await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) + else: + await interaction_or_msg.edit(embed=self.embed, view=new_view) + except discord.errors.HTTPException: # Various errors such as embed description being too long. + pass + else: + self.stop() + + def copy(self) -> FilterListAddView: + """Create a copy of this view.""" + return FilterListAddView( + self.list_name, + self.list_type, + self.settings, + self.loaded_settings, + self.author, + self.embed, + self.confirm_callback + ) + + class FilterListEditView(EditBaseView): - """A view used to edit a filter's settings before updating the database.""" + """A view used to edit a filter list's settings before updating the database.""" def __init__( self, diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index dc3bd01c9f..0ded6d4a4b 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -266,18 +266,17 @@ async def on_submit(self, interaction: Interaction) -> None: """Send the submitted value to be added to the list.""" await self.view.apply_edit(interaction, self.new_value.value) - def __init__(self, setting_name: str, starting_value: list, type_: type, update_callback: Callable): + def __init__(self, setting_name: str, starting_value: list, update_callback: Callable): super().__init__(timeout=COMPONENT_TIMEOUT) self.setting_name = setting_name self.stored_value = starting_value - self.type_ = type_ self.update_callback = update_callback - options = [SelectOption(label=item) for item in starting_value[:MAX_SELECT_ITEMS]] + options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] self.removal_select = CustomCallbackSelect( self.apply_removal, placeholder="Enter an item to remove", options=options, row=1 ) - if starting_value: + if self.stored_value: self.add_item(self.removal_select) async def apply_removal(self, interaction: Interaction, select: discord.ui.Select) -> None: @@ -391,12 +390,12 @@ async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.S view = BooleanSelectView(setting_name, update_callback) await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) elif type_ in (set, list, tuple): - current_list = self.current_value(setting_name) + current_list = list(self.current_value(setting_name)) if current_list is MISSING: current_list = [] await interaction.response.send_message( f"Current list: {current_list}", - view=SequenceEditView(setting_name, current_list, type_, update_callback), + view=SequenceEditView(setting_name, current_list, update_callback), ephemeral=True ) elif isinstance(type_, EnumMeta): diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index 438c22d416..7149f7254c 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -4,7 +4,7 @@ import pkgutil from abc import ABC, abstractmethod from collections import defaultdict -from typing import Any, Iterable, Union +from typing import Any, Iterable, TypeVar, Union import regex @@ -13,6 +13,9 @@ ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIATION_SELECTORS}]]", regex.V1) +T = TypeVar('T') + + def subclasses_in_package(package: str, prefix: str, parent: type) -> set[type]: """Return all the subclasses of class `parent`, found in the top-level of `package`, given by absolute path.""" subclasses = set() @@ -82,6 +85,22 @@ def repr_equals(override: Any, default: Any) -> bool: return str(override) == str(default) +def starting_value(type_: type[T]) -> T: + """Return a value of the given type.""" + if hasattr(type_, "__origin__"): + if type_.__origin__ is not Union: # In case this is a types.GenericAlias or a typing._GenericAlias + type_ = type_.__origin__ + if hasattr(type_, "__args__"): # In case of a Union + if type(None) in type_.__args__: + return None + type_ = type_.__args__[0] # Pick one, doesn't matter + + try: + return type_() + except TypeError: # In case it all fails, return a string and let the user handle it. + return "" + + class FieldRequiring(ABC): """A mixin class that can force its concrete subclasses to set a value for specific class attributes.""" diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index a906b6a413..09f458a59d 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -23,9 +23,11 @@ from bot.exts.filtering._ui.filter import ( build_filter_repr_dict, description_and_settings_converter, filter_overrides, populate_embed_from_dict ) -from bot.exts.filtering._ui.filter_list import DeleteConfirmationView, FilterListEditView, settings_converter +from bot.exts.filtering._ui.filter_list import ( + DeleteConfirmationView, FilterListAddView, FilterListEditView, settings_converter +) from bot.exts.filtering._ui.ui import ArgumentCompletionView -from bot.exts.filtering._utils import past_tense, to_serializable +from bot.exts.filtering._utils import past_tense, starting_value, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.messages import format_channel, format_user @@ -36,6 +38,9 @@ class Filtering(Cog): """Filtering and alerting for content posted on the server.""" + # A set of filter list names with missing implementations that already caused a warning. + already_warned = set() + # region: init def __init__(self, bot: Bot): @@ -55,21 +60,10 @@ async def cog_load(self) -> None: Additionally, fetch the alerting webhook. """ await self.bot.wait_until_guild_available() - already_warned = set() raw_filter_lists = await self.bot.api_client.get("bot/filter/filter_lists") for raw_filter_list in raw_filter_lists: - list_name = raw_filter_list["name"] - if list_name not in self.filter_lists: - if list_name not in filter_list_types: - if list_name not in already_warned: - log.warning( - f"A filter list named {list_name} was loaded from the database, but no matching class." - ) - already_warned.add(list_name) - continue - self.filter_lists[list_name] = filter_list_types[list_name](self) - self.filter_lists[list_name].add_list(raw_filter_list) + self._load_raw_filter_list(raw_filter_list) try: self.webhook = await self.bot.fetch_webhook(Webhooks.filters) @@ -520,10 +514,36 @@ async def fl_describe( # Use the class's docstring, and ignore single newlines. embed.description = re.sub(r"(? None: + """Add a new filter list.""" + list_description = f"{past_tense(list_type.name.lower())} {list_name.lower()}" + if list_name in self.filter_lists: + filter_list = self.filter_lists[list_name] + if list_type in filter_list.filter_lists: + await ctx.reply(f":x: The {list_description} filter list already exists.") + return + + embed = Embed(colour=Colour.blue()) + embed.set_author(name=f"New Filter List - {list_description.title()}") + settings = {name: starting_value(value[2]) for name, value in self.loaded_settings.items()} + + view = FilterListAddView( + list_name, + list_type, + settings, + self.loaded_settings, + ctx.author, + embed, + self._post_filter_list + ) + await ctx.send(embed=embed, reference=ctx.message, view=view) + @filterlist.command(name="edit", aliases=("e",)) @has_any_role(Roles.admins) async def fl_edit( @@ -600,6 +620,20 @@ async def delete_list() -> None: # endregion # region: helper functions + def _load_raw_filter_list(self, list_data: dict) -> None: + """Load the raw list data to the cog.""" + list_name = list_data["name"] + if list_name not in self.filter_lists: + if list_name not in filter_list_types: + if list_name not in self.already_warned: + log.warning( + f"A filter list named {list_name} was loaded from the database, but no matching class." + ) + self.already_warned.add(list_name) + return + self.filter_lists[list_name] = filter_list_types[list_name](self) + self.filter_lists[list_name].add_list(list_data) + async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], dict[FilterList, str]]: """ Return the actions that should be taken for all filter lists in the given context. @@ -844,16 +878,27 @@ async def _patch_filter( payload = { "content": content, "description": description, "additional_field": json.dumps(filter_settings), **settings } - response = await bot.instance.api_client.patch(f'bot/filter/filters/{filter_.id}', json=payload) + response = await bot.instance.api_client.patch( + f'bot/filter/filters/{filter_.id}', json=to_serializable(payload) + ) edited_filter = filter_list.add_filter(response, list_type) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, edited_filter) await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) + async def _post_filter_list(self, msg: Message, list_name: str, list_type: ListType, settings: dict) -> None: + """POST the new data of the filter list to the site API.""" + payload = {"name": list_name, "list_type": list_type.value, **to_serializable(settings)} + response = await bot.instance.api_client.post('bot/filter/filter_lists', json=payload) + self._load_raw_filter_list(response) + await msg.reply(f"✅ Added a new filter list: {past_tense(list_type.name.lower())} {list_name}") + @staticmethod async def _patch_filter_list(msg: Message, filter_list: FilterList, list_type: ListType, settings: dict) -> None: """PATCH the new data of the filter list to the site API.""" list_id = filter_list.list_ids[list_type] - response = await bot.instance.api_client.patch(f'bot/filter/filter_lists/{list_id}', json=settings) + response = await bot.instance.api_client.patch( + f'bot/filter/filter_lists/{list_id}', json=to_serializable(settings) + ) filter_list.remove_list(list_type) filter_list.add_list(response) await msg.reply(f"✅ Edited filter list: {past_tense(list_type.name.lower())} {filter_list.name}") From a4fba43d402c6dd4df99748a2c9a81302ad54ace Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 11 Oct 2022 13:23:48 +0300 Subject: [PATCH 046/132] Rearrange FilterList class --- bot/exts/filtering/_filter_lists/domain.py | 6 +- bot/exts/filtering/_filter_lists/extension.py | 10 +- .../filtering/_filter_lists/filter_list.py | 104 +++++++++++++----- bot/exts/filtering/_filter_lists/invite.py | 12 +- bot/exts/filtering/_filter_lists/token.py | 6 +- bot/exts/filtering/_ui/filter.py | 4 +- bot/exts/filtering/_ui/filter_list.py | 4 +- bot/exts/filtering/_ui/ui.py | 5 +- bot/exts/filtering/filtering.py | 101 +++++++++-------- 9 files changed, 153 insertions(+), 99 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index cae2eb8782..ec43e92dff 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -33,7 +33,7 @@ class DomainsList(FilterList): name = "domain" def __init__(self, filtering_cog: Filtering): - super().__init__(DomainFilter) + super().__init__() filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) def get_filter_type(self, content: str) -> Type[Filter]: @@ -56,13 +56,13 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings new_ctx = ctx.replace(content=urls) triggers = self.filter_list_result( - new_ctx, self.filter_lists[ListType.DENY], self.defaults[ListType.DENY]["validations"] + new_ctx, self[ListType.DENY].filters, self[ListType.DENY].defaults.validations ) ctx.notification_domain = new_ctx.notification_domain actions = None message = "" if triggers: - action_defaults = self.defaults[ListType.DENY]["actions"] + action_defaults = self[ListType.DENY].defaults.actions actions = reduce( or_, (filter_.actions.fallback_to(action_defaults) if filter_.actions else action_defaults diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index e34ead3935..ce1a46e4aa 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -49,7 +49,7 @@ class ExtensionsList(FilterList): name = "extension" def __init__(self, filtering_cog: Filtering): - super().__init__(ExtensionFilter) + super().__init__() filtering_cog.subscribe(self, Event.MESSAGE) self._whitelisted_description = None @@ -68,7 +68,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings if not ctx.message.attachments: return None, "" - _, failed = self.defaults[ListType.ALLOW]["validations"].evaluate(ctx) + _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) if failed: # There's no extension filtering in this context. return None, "" @@ -77,7 +77,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings (splitext(attachment.filename.lower())[1], attachment.filename) for attachment in ctx.message.attachments } new_ctx = ctx.replace(content={ext for ext, _ in all_ext}) # And prepare the context for the filters to read. - triggered = [filter_ for filter_ in self.filter_lists[ListType.ALLOW].values() if filter_.triggered_on(new_ctx)] + triggered = [filter_ for filter_ in self[ListType.ALLOW].filters.values() if filter_.triggered_on(new_ctx)] allowed_ext = {filter_.content for filter_ in triggered} # Get the extensions in the message that are allowed. # See if there are any extensions left which aren't allowed. @@ -101,7 +101,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings meta_channel = bot.instance.get_channel(Channels.meta) if not self._whitelisted_description: self._whitelisted_description = ', '.join( - filter_.content for filter_ in self.filter_lists[ListType.ALLOW].values() + filter_.content for filter_ in self[ListType.ALLOW].filters.values() ) ctx.dm_embed = DISALLOWED_EMBED_DESCRIPTION.format( joined_whitelist=self._whitelisted_description, @@ -110,4 +110,4 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings ) ctx.matches += not_allowed.values() - return self.defaults[ListType.ALLOW]["actions"], ", ".join(f"`{ext}`" for ext in not_allowed) + return self[ListType.ALLOW].defaults.actions, ", ".join(f"`{ext}`" for ext in not_allowed) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index a4f22aed47..ecbcb8f091 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,6 +1,7 @@ from abc import abstractmethod +from collections.abc import Iterator from enum import Enum -from typing import Any, Optional, Type +from typing import Any, ItemsView, NamedTuple, Optional, Type from discord.ext.commands import BadArgument @@ -36,6 +37,32 @@ def list_type_converter(argument: str) -> ListType: raise BadArgument(f"No matching list type found for {argument!r}.") +class Defaults(NamedTuple): + """Represents an atomic list's default settings.""" + + actions: ActionSettings + validations: ValidationSettings + + +class AtomicList(NamedTuple): + """ + Represents the atomic structure of a single filter list as it appears in the database. + + This is as opposed to the FilterList class which is a combination of several list types. + """ + + id: int + name: str + list_type: ListType + defaults: Defaults + filters: dict[int, Filter] + + @property + def label(self) -> str: + """Provide a short description identifying the list with its name and type.""" + return f"{past_tense(self.list_type.name.lower())} {self.name.lower()}" + + class FilterList(FieldRequiring): """Dispatches events to lists of _filters, and aggregates the responses into a single list of actions to take.""" @@ -43,52 +70,63 @@ class FilterList(FieldRequiring): # Names must be unique across all filter lists. name = FieldRequiring.MUST_SET_UNIQUE - def __init__(self, filter_type: Type[Filter]): - self.list_ids = {} - self.filter_lists: dict[ListType, dict[int, Filter]] = {} - self.defaults = {} + def __init__(self): + self._filter_lists: dict[ListType, AtomicList] = {} + + def __iter__(self) -> Iterator[ListType]: + return iter(self._filter_lists) + + def __getitem__(self, list_type: ListType) -> AtomicList: + return self._filter_lists[list_type] + + def __contains__(self, list_type: ListType) -> bool: + return list_type in self._filter_lists - self.filter_type = filter_type + def __bool__(self) -> bool: + return bool(self._filter_lists) - def add_list(self, list_data: dict) -> None: + def __len__(self) -> int: + return len(self._filter_lists) + + def items(self) -> ItemsView[ListType, AtomicList]: + """Return an iterator for the lists' types and values.""" + return self._filter_lists.items() + + def add_list(self, list_data: dict) -> AtomicList: """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" actions, validations = create_settings(list_data["settings"], keep_empty=True) list_type = ListType(list_data["list_type"]) - self.defaults[list_type] = {"actions": actions, "validations": validations} - self.list_ids[list_type] = list_data["id"] + defaults = Defaults(actions, validations) - self.filter_lists[list_type] = {} + filters = {} for filter_data in list_data["filters"]: - self.add_filter(filter_data, list_type) + filters[filter_data["id"]] = self._create_filter(filter_data) + + self._filter_lists[list_type] = AtomicList(list_data["id"], self.name, list_type, defaults, filters) + return self._filter_lists[list_type] def remove_list(self, list_type: ListType) -> None: """Remove the list associated with the given type from the FilterList object.""" - if list_type not in self.filter_lists: + if list_type not in self._filter_lists: return - self.filter_lists.pop(list_type) - self.defaults.pop(list_type) - self.list_ids.pop(list_type) - - def add_filter(self, filter_data: dict, list_type: ListType) -> Filter: - """Add a filter to the list of the specified type.""" - try: - new_filter = self.filter_type(filter_data) - self.filter_lists[list_type][filter_data["id"]] = new_filter - except TypeError as e: - log.warning(e) - else: - return new_filter + self._filter_lists.pop(list_type) def default(self, list_type: ListType, setting: str) -> Any: """Get the default value of a specific setting.""" missing = object() - value = self.defaults[list_type]["actions"].get_setting(setting, missing) + value = self._filter_lists[list_type].defaults.actions.get_setting(setting, missing) if value is missing: - value = self.defaults[list_type]["validations"].get_setting(setting, missing) + value = self._filter_lists[list_type].defaults.validations.get_setting(setting, missing) if value is missing: raise ValueError(f"Could find a setting named {setting}.") return value + def add_filter(self, list_type: ListType, filter_data: dict) -> Filter: + """Add a filter to the list of the specified type.""" + new_filter = self._create_filter(filter_data) + self[list_type].filters[filter_data["id"]] = new_filter + return new_filter + @abstractmethod def get_filter_type(self, content: str) -> Type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" @@ -104,7 +142,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings @staticmethod def filter_list_result( - ctx: FilterContext, filters: dict[int, Filter], defaults: ValidationSettings + ctx: FilterContext, filters: dict[int, Filter], defaults: ValidationSettings ) -> list[Filter]: """ Sift through the list of filters, and return only the ones which apply to the given context. @@ -134,3 +172,13 @@ def filter_list_result( relevant_filters.append(filter_) return relevant_filters + + def _create_filter(self, filter_data: dict) -> Filter: + """Create a filter from the given data.""" + try: + filter_type = self.get_filter_type(filter_data["content"]) + new_filter = filter_type(filter_data) + except TypeError as e: + log.warning(e) + else: + return new_filter diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 095699597c..30884a2ab9 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -39,7 +39,7 @@ class InviteList(FilterList): name = "invite" def __init__(self, filtering_cog: Filtering): - super().__init__(InviteFilter) + super().__init__() filtering_cog.subscribe(self, Event.MESSAGE) def get_filter_type(self, content: str) -> Type[Filter]: @@ -53,7 +53,7 @@ def filter_types(self) -> set[Type[Filter]]: async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" - _, failed = self.defaults[ListType.ALLOW]["validations"].evaluate(ctx) + _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) if failed: # There's no invite filtering in this context. return None, "" @@ -89,7 +89,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings guilds_for_inspection = {invite.guild.id for invite in denied_by_default.values()} new_ctx = ctx.replace(content=guilds_for_inspection) allowed = { - filter_.content for filter_ in self.filter_lists[ListType.ALLOW].values() if filter_.triggered_on(new_ctx) + filter_.content for filter_ in self[ListType.ALLOW].filters.values() if filter_.triggered_on(new_ctx) } disallowed_invites.update({ invite_code: invite for invite_code, invite in denied_by_default.items() if invite.guild.id not in allowed @@ -99,7 +99,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings guilds_for_inspection = {invite.guild.id for invite in allowed_by_default.values()} new_ctx = ctx.replace(content=guilds_for_inspection) triggered = self.filter_list_result( - new_ctx, self.filter_lists[ListType.ALLOW], self.defaults[ListType.DENY]["validations"] + new_ctx, self[ListType.ALLOW].filters, self[ListType.DENY].defaults.validations ) disallowed_invites.update({ invite_code: invite for invite_code, invite in allowed_by_default.items() @@ -111,14 +111,14 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings actions = None if len(disallowed_invites) > len(triggered): # There are invites which weren't allowed but aren't blacklisted. - deny_defaults = self.defaults[ListType.DENY]["actions"] + deny_defaults = self[ListType.DENY].defaults.actions actions = reduce( or_, ( filter_.actions.fallback_to(deny_defaults) if filter_.actions else deny_defaults for filter_ in triggered ), - self.defaults[ListType.ALLOW]["actions"] + self[ListType.ALLOW].defaults.actions ) elif triggered: actions = reduce(or_, (filter_.actions for filter_ in triggered)) diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index aca0fdedf2..2abf945534 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -34,7 +34,7 @@ class TokensList(FilterList): name = "token" def __init__(self, filtering_cog: Filtering): - super().__init__(TokenFilter) + super().__init__() filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) def get_filter_type(self, content: str) -> Type[Filter]: @@ -57,12 +57,12 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings ctx = ctx.replace(content=text) triggers = self.filter_list_result( - ctx, self.filter_lists[ListType.DENY], self.defaults[ListType.DENY]["validations"] + ctx, self[ListType.DENY].filters, self[ListType.DENY].defaults.validations ) actions = None message = "" if triggers: - action_defaults = self.defaults[ListType.DENY]["actions"] + action_defaults = self[ListType.DENY].defaults.actions actions = reduce( or_, (filter_.actions.fallback_to(action_defaults) if filter_.actions else action_defaults diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index 4da8fe0019..b372cac3ee 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -31,8 +31,8 @@ def build_filter_repr_dict( """Build a dictionary of field names and values to pass to `_build_embed_from_dict`.""" # Get filter list settings default_setting_values = {} - for type_ in ("actions", "validations"): - for _, setting in filter_list.defaults[list_type][type_].items(): + for settings_group in filter_list[list_type].defaults: + for _, setting in settings_group.items(): default_setting_values.update(to_serializable(setting.dict())) # Add overrides. It's done in this way to preserve field order, since the filter won't have all settings. diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index cc58f5a62e..55e03ce18f 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -46,8 +46,8 @@ def build_filterlist_repr_dict(filter_list: FilterList, list_type: ListType, new """Build a dictionary of field names and values to pass to `_build_embed_from_dict`.""" # Get filter list settings default_setting_values = {} - for type_ in ("actions", "validations"): - for _, setting in filter_list.defaults[list_type][type_].items(): + for settings_group in filter_list[list_type].defaults: + for _, setting in settings_group.items(): default_setting_values.update(to_serializable(setting.dict())) # Add new values. It's done in this way to preserve field order, since the new_values won't have all settings. diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 0ded6d4a4b..cba151c705 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -390,8 +390,9 @@ async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.S view = BooleanSelectView(setting_name, update_callback) await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) elif type_ in (set, list, tuple): - current_list = list(self.current_value(setting_name)) - if current_list is MISSING: + if (current_value := self.current_value(setting_name)) is not MISSING: + current_list = list(current_value) + else: current_list = [] await interaction.response.send_message( f"Current list: {current_list}", diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 09f458a59d..607fbe8bfb 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -18,6 +18,7 @@ from bot.constants import Colours, MODERATION_ROLES, Roles, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter +from bot.exts.filtering._filter_lists.filter_list import AtomicList from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._ui.filter import ( @@ -62,15 +63,18 @@ async def cog_load(self) -> None: await self.bot.wait_until_guild_available() raw_filter_lists = await self.bot.api_client.get("bot/filter/filter_lists") + example_list = None for raw_filter_list in raw_filter_lists: - self._load_raw_filter_list(raw_filter_list) + loaded_list = self._load_raw_filter_list(raw_filter_list) + if not example_list and loaded_list: + example_list = loaded_list try: self.webhook = await self.bot.fetch_webhook(Webhooks.filters) except HTTPException: log.error(f"Failed to fetch filters webhook with ID `{Webhooks.filters}`.") - self.collect_loaded_types() + self.collect_loaded_types(example_list) def subscribe(self, filter_list: FilterList, *events: Event) -> None: """ @@ -98,12 +102,14 @@ def unsubscribe(self, filter_list: FilterList, *events: Event) -> None: if filter_list in self._subscriptions.get(event, []): self._subscriptions[event].remove(filter_list) - def collect_loaded_types(self) -> None: + def collect_loaded_types(self, example_list: AtomicList) -> None: """ Go over the classes used in initialization and collect them to dictionaries. The information that is collected is about the types actually used to load the API response, not all types available in the filtering extension. + + Any filter list has the fields for all settings in the DB schema, so picking any one of them is enough. """ # Get the filter types used by each filter list. for filter_list in self.filter_lists.values(): @@ -111,26 +117,25 @@ def collect_loaded_types(self) -> None: # Get the setting types used by each filter list. if self.filter_lists: - # Any filter list has the fields for all settings in the DB schema, so picking any one of them is enough. - list_defaults = list(list(self.filter_lists.values())[0].defaults.values())[0] - settings_types = set() + settings_entries = set() # The settings are split between actions and validations. - settings_types.update(type(setting) for _, setting in list_defaults["actions"].items()) - settings_types.update(type(setting) for _, setting in list_defaults["validations"].items()) - for setting_type in settings_types: - type_hints = get_type_hints(setting_type) + for settings_group in example_list.defaults: + settings_entries.update(type(setting) for _, setting in settings_group.items()) + + for setting_entry in settings_entries: + type_hints = get_type_hints(setting_entry) # The description should be either a string or a dictionary. - if isinstance(setting_type.description, str): - # If it's a string, then the setting matches a single field in the DB, + if isinstance(setting_entry.description, str): + # If it's a string, then the settings entry matches a single field in the DB, # and its name is the setting type's name attribute. - self.loaded_settings[setting_type.name] = ( - setting_type.description, setting_type, type_hints[setting_type.name] + self.loaded_settings[setting_entry.name] = ( + setting_entry.description, setting_entry, type_hints[setting_entry.name] ) else: - # Otherwise, the setting type works with compound settings. + # Otherwise, the setting entry works with compound settings. self.loaded_settings.update({ - subsetting: (description, setting_type, type_hints[subsetting]) - for subsetting, description in setting_type.description.items() + subsetting: (description, setting_entry, type_hints[subsetting]) + for subsetting, description in setting_entry.description.items() }) # Get the settings per filter as well. @@ -286,7 +291,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: embed.description = f"`{filter_.content}`" if filter_.description: embed.description += f" - {filter_.description}" - embed.set_author(name=f"Filter #{id_} - " + f"{past_tense(list_type.name.lower())} {filter_list.name}".title()) + embed.set_author(name=f"Filter #{id_} - " + f"{filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." @@ -410,7 +415,7 @@ async def f_edit( if description: embed.description += f" - {description}" embed.set_author( - name=f"Filter #{filter_id} - {past_tense(list_type.name.lower())} {filter_list.name}".title()) + name=f"Filter #{filter_id} - {filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." @@ -441,7 +446,7 @@ async def f_delete(self, ctx: Context, filter_id: int) -> None: return filter_, filter_list, list_type = result await bot.instance.api_client.delete(f'bot/filter/filters/{filter_id}') - filter_list.filter_lists[list_type].pop(filter_id) + filter_list[list_type].filters.pop(filter_id) await ctx.reply(f"✅ Deleted filter: {filter_}") @filter.group(aliases=("settings",)) @@ -503,10 +508,9 @@ async def fl_describe( return list_type, filter_list = result - list_defaults = filter_list.defaults[list_type] setting_values = {} - for type_ in ("actions", "validations"): - for _, setting in list_defaults[type_].items(): + for settings_group in filter_list[list_type].defaults: + for _, setting in settings_group.items(): setting_values.update(to_serializable(setting.dict())) embed = Embed(colour=Colour.blue()) @@ -514,7 +518,7 @@ async def fl_describe( # Use the class's docstring, and ignore single newlines. embed.description = re.sub(r"(? None: message = await ctx.send("⏳ Annihilation in progress, please hold...", file=file) # Unload the filter list. filter_list.remove_list(list_type) - if not filter_list.filter_lists: # There's nothing left, remove from the cog. + if not filter_list: # There's nothing left, remove from the cog. self.filter_lists.pop(filter_list.name) self.unsubscribe(filter_list) @@ -610,8 +614,8 @@ async def delete_list() -> None: if result is None: return list_type, filter_list = result - list_id = filter_list.list_ids[list_type] - list_description = f"{past_tense(list_type.name.lower())} {filter_list.name}" + list_id = filter_list[list_type].id + list_description = filter_list[list_type].label await ctx.reply( f"Are you sure you want to delete the {list_description} list?", view=DeleteConfirmationView(ctx.author, delete_list) @@ -620,7 +624,7 @@ async def delete_list() -> None: # endregion # region: helper functions - def _load_raw_filter_list(self, list_data: dict) -> None: + def _load_raw_filter_list(self, list_data: dict) -> AtomicList | None: """Load the raw list data to the cog.""" list_name = list_data["name"] if list_name not in self.filter_lists: @@ -630,9 +634,9 @@ def _load_raw_filter_list(self, list_data: dict) -> None: f"A filter list named {list_name} was loaded from the database, but no matching class." ) self.already_warned.add(list_name) - return + return None self.filter_lists[list_name] = filter_list_types[list_name](self) - self.filter_lists[list_name].add_list(list_data) + return self.filter_lists[list_name].add_list(list_data) async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], dict[FilterList, str]]: """ @@ -703,7 +707,7 @@ async def _resolve_list_type_and_name( filter_list = self._get_list_by_name(list_name) if list_type is None: - if len(filter_list.filter_lists) > 1: + if len(filter_list) > 1: await ctx.send( "The **list_type** argument is unspecified. Please pick a value from the options below:", view=ArgumentCompletionView( @@ -711,7 +715,7 @@ async def _resolve_list_type_and_name( ) ) return None - list_type = list(filter_list.filter_lists)[0] + list_type = list(filter_list)[0] return list_type, filter_list def _get_list_by_name(self, list_name: str) -> FilterList: @@ -729,25 +733,24 @@ def _get_list_by_name(self, list_name: str) -> FilterList: @staticmethod async def _send_list(ctx: Context, filter_list: FilterList, list_type: ListType) -> None: """Show the list of filters identified by the list name and type.""" - type_filters = filter_list.filter_lists.get(list_type) - if type_filters is None: + if list_type not in filter_list: await ctx.send(f":x: There is no list of {past_tense(list_type.name.lower())} {filter_list.name}s.") return - lines = list(map(str, type_filters.values())) + lines = list(map(str, filter_list[list_type].filters.values())) log.trace(f"Sending a list of {len(lines)} filters.") embed = Embed(colour=Colour.blue()) - embed.set_author(name=f"List of {past_tense(list_type.name.lower())} {filter_list.name}s ({len(lines)} total)") + embed.set_author(name=f"List of {filter_list[list_type].label}s ({len(lines)} total)") await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) def _get_filter_by_id(self, id_: int) -> Optional[tuple[Filter, FilterList, ListType]]: """Get the filter object corresponding to the provided ID, along with its containing list and list type.""" for filter_list in self.filter_lists.values(): - for list_type, sublist in filter_list.filter_lists.items(): - if id_ in sublist: - return sublist[id_], filter_list, list_type + for list_type, sublist in filter_list.items(): + if id_ in sublist.filters: + return sublist.filters[id_], filter_list, list_type async def _add_filter( self, @@ -783,7 +786,7 @@ async def _add_filter( if description: embed.description += f" - {description}" embed.set_author( - name=f"New Filter - {past_tense(list_type.name.lower())} {filter_list.name}".title()) + name=f"New Filter - {filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." @@ -808,8 +811,10 @@ async def _add_filter( @staticmethod def _identical_filters_message(content: str, filter_list: FilterList, list_type: ListType, filter_: Filter) -> str: """Returns all the filters in the list with content identical to the content supplied.""" + if list_type not in filter_list: + return "" duplicates = [ - f for f in filter_list.filter_lists.get(list_type, {}).values() + f for f in filter_list[list_type].filters.values() if f.content == content and f.id != filter_.id ] msg = "" @@ -837,14 +842,14 @@ async def _post_new_filter( content = await filter_type.process_content(content) - list_id = filter_list.list_ids[list_type] + list_id = filter_list[list_type].id description = description or None payload = { "filter_list": list_id, "content": content, "description": description, "additional_field": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.post('bot/filter/filters', json=to_serializable(payload)) - new_filter = filter_list.add_filter(response, list_type) + new_filter = filter_list.add_filter(list_type, response) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, new_filter) await msg.reply(f"✅ Added filter: {new_filter}" + extra_msg) @@ -881,7 +886,7 @@ async def _patch_filter( response = await bot.instance.api_client.patch( f'bot/filter/filters/{filter_.id}', json=to_serializable(payload) ) - edited_filter = filter_list.add_filter(response, list_type) + edited_filter = filter_list.add_filter(list_type, response) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, edited_filter) await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) @@ -895,13 +900,13 @@ async def _post_filter_list(self, msg: Message, list_name: str, list_type: ListT @staticmethod async def _patch_filter_list(msg: Message, filter_list: FilterList, list_type: ListType, settings: dict) -> None: """PATCH the new data of the filter list to the site API.""" - list_id = filter_list.list_ids[list_type] + list_id = filter_list[list_type].id response = await bot.instance.api_client.patch( f'bot/filter/filter_lists/{list_id}', json=to_serializable(settings) ) filter_list.remove_list(list_type) filter_list.add_list(response) - await msg.reply(f"✅ Edited filter list: {past_tense(list_type.name.lower())} {filter_list.name}") + await msg.reply(f"✅ Edited filter list: {filter_list[list_type].label}") # endregion From ffcbf77c27e75215caca7f8dddb74a6a76f18d8b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 11 Oct 2022 15:01:17 +0300 Subject: [PATCH 047/132] Add confirmation before filter deletion --- bot/exts/filtering/_ui/filter_list.py | 29 +-------------------------- bot/exts/filtering/_ui/ui.py | 26 ++++++++++++++++++++++++ bot/exts/filtering/filtering.py | 19 +++++++++++------- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index 55e03ce18f..b072e293ed 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -4,7 +4,7 @@ import discord from botcore.site_api import ResponseCodeError -from discord import Embed, Interaction, Member, SelectOption, User +from discord import Embed, Interaction, SelectOption, User from discord.ext.commands import BadArgument from bot.exts.filtering._filter_lists import FilterList, ListType @@ -14,9 +14,6 @@ ) from bot.exts.filtering._utils import repr_equals, to_serializable -# Amount of seconds to confirm the operation. -DELETION_TIMEOUT = 60 - def settings_converter(loaded_settings: dict, input_data: str) -> dict[str, Any]: """Parse a string representing settings, and validate the setting names.""" @@ -268,27 +265,3 @@ def copy(self) -> FilterListEditView: self.embed, self.confirm_callback ) - - -class DeleteConfirmationView(discord.ui.View): - """A view to confirm the deletion of a filter list.""" - - def __init__(self, author: Member | User, callback: Callable): - super().__init__(timeout=DELETION_TIMEOUT) - self.author = author - self.callback = callback - - async def interaction_check(self, interaction: Interaction) -> bool: - """Only allow interactions from the command invoker.""" - return interaction.user.id == self.author.id - - @discord.ui.button(label="Delete", style=discord.ButtonStyle.red, row=0) - async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: - """Invoke the filter list deletion.""" - await interaction.response.edit_message(view=None) - await self.callback() - - @discord.ui.button(label="Cancel", row=0) - async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: - """Cancel the filter list deletion.""" - await interaction.response.edit_message(content="🚫 Operation canceled.", view=None) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index cba151c705..5a60bb21e6 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -25,6 +25,8 @@ EDIT_TIMEOUT = 600 # Number of seconds before timeout of an editing component. COMPONENT_TIMEOUT = 180 +# Amount of seconds to confirm the operation. +DELETION_TIMEOUT = 60 # Max length of modal title MAX_MODAL_TITLE_LENGTH = 45 # Max number of items in a select @@ -421,3 +423,27 @@ async def update_embed(self, interaction_or_msg: Interaction | discord.Message) @abstractmethod def copy(self) -> EditBaseView: """Create a copy of this view.""" + + +class DeleteConfirmationView(discord.ui.View): + """A view to confirm a deletion.""" + + def __init__(self, author: discord.Member | discord.User, callback: Callable): + super().__init__(timeout=DELETION_TIMEOUT) + self.author = author + self.callback = callback + + async def interaction_check(self, interaction: Interaction) -> bool: + """Only allow interactions from the command invoker.""" + return interaction.user.id == self.author.id + + @discord.ui.button(label="Delete", style=discord.ButtonStyle.red, row=0) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Invoke the filter list deletion.""" + await interaction.response.edit_message(view=None) + await self.callback() + + @discord.ui.button(label="Cancel", row=0) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the filter list deletion.""" + await interaction.response.edit_message(content="🚫 Operation canceled.", view=None) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 607fbe8bfb..6bc6659739 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -24,10 +24,8 @@ from bot.exts.filtering._ui.filter import ( build_filter_repr_dict, description_and_settings_converter, filter_overrides, populate_embed_from_dict ) -from bot.exts.filtering._ui.filter_list import ( - DeleteConfirmationView, FilterListAddView, FilterListEditView, settings_converter -) -from bot.exts.filtering._ui.ui import ArgumentCompletionView +from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter +from bot.exts.filtering._ui.ui import ArgumentCompletionView, DeleteConfirmationView from bot.exts.filtering._utils import past_tense, starting_value, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator @@ -440,14 +438,21 @@ async def f_edit( @filter.command(name="delete", aliases=("d", "remove")) async def f_delete(self, ctx: Context, filter_id: int) -> None: """Delete the filter specified by its ID.""" + async def delete_list() -> None: + """The actual removal routine.""" + await bot.instance.api_client.delete(f'bot/filter/filters/{filter_id}') + filter_list[list_type].filters.pop(filter_id) + await ctx.reply(f"✅ Deleted filter: {filter_}") + result = self._get_filter_by_id(filter_id) if result is None: await ctx.send(f":x: Could not find a filter with ID `{filter_id}`.") return filter_, filter_list, list_type = result - await bot.instance.api_client.delete(f'bot/filter/filters/{filter_id}') - filter_list[list_type].filters.pop(filter_id) - await ctx.reply(f"✅ Deleted filter: {filter_}") + await ctx.reply( + f"Are you sure you want to delete filter {filter_}?", + view=DeleteConfirmationView(ctx.author, delete_list) + ) @filter.group(aliases=("settings",)) async def setting(self, ctx: Context) -> None: From a9d1371cc93fdfc60aa1fcc482840cf8dbcd3ae2 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 11 Oct 2022 16:14:47 +0300 Subject: [PATCH 048/132] Better API error handling --- bot/exts/filtering/_filters/domain.py | 3 ++- bot/exts/filtering/_filters/filter.py | 2 +- bot/exts/filtering/_filters/invite.py | 7 ++++--- bot/exts/filtering/_filters/token.py | 4 +++- bot/exts/filtering/_ui/filter.py | 4 ++-- bot/exts/filtering/filtering.py | 20 +++++++++++++++----- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index eed2b67213..4976198cd9 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse import tldextract +from discord.ext.commands import BadArgument from pydantic import BaseModel from bot.exts.filtering._filter_context import FilterContext @@ -57,5 +58,5 @@ async def process_content(cls, content: str) -> str: """ match = URL_RE.fullmatch(content) if not match or not match.group(1): - raise ValueError(f"`{content}` is not a URL.") + raise BadArgument(f"`{content}` is not a URL.") return match.group(1) diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 957957d834..b4a2bfe5e2 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -53,7 +53,7 @@ async def process_content(cls, content: str) -> str: """ Process the content into a form which will work with the filtering. - A ValueError should be raised if the content can't be used. + A BadArgument should be raised if the content can't be used. """ return content diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py index e0f4695206..0463b0032a 100644 --- a/bot/exts/filtering/_filters/invite.py +++ b/bot/exts/filtering/_filters/invite.py @@ -1,5 +1,6 @@ from botcore.utils.regex import DISCORD_INVITE from discord import NotFound +from discord.ext.commands import BadArgument import bot from bot.exts.filtering._filter_context import FilterContext @@ -32,12 +33,12 @@ async def process_content(cls, content: str) -> str: """ match = DISCORD_INVITE.fullmatch(content) if not match or not match.group("invite"): - raise ValueError(f"`{content}` is not a valid Discord invite.") + raise BadArgument(f"`{content}` is not a valid Discord invite.") invite_code = match.group("invite") try: invite = await bot.instance.fetch_invite(invite_code) except NotFound: - raise ValueError(f"`{invite_code}` is not a valid Discord invite code.") + raise BadArgument(f"`{invite_code}` is not a valid Discord invite code.") if not invite.guild: - raise ValueError("Did you just try to add a group DM?") + raise BadArgument("Did you just try to add a group DM?") return str(invite.guild.id) diff --git a/bot/exts/filtering/_filters/token.py b/bot/exts/filtering/_filters/token.py index a4c646c5a7..04e30cb039 100644 --- a/bot/exts/filtering/_filters/token.py +++ b/bot/exts/filtering/_filters/token.py @@ -1,5 +1,7 @@ import re +from discord.ext.commands import BadArgument + from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter @@ -29,5 +31,5 @@ async def process_content(cls, content: str) -> str: try: re.compile(content) except re.error as e: - raise ValueError(str(e)) + raise BadArgument(str(e)) return content diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index b372cac3ee..e6a568ad0b 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -214,9 +214,9 @@ async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> except ResponseCodeError as e: await interaction.message.reply(embed=format_response_error(e)) await interaction.message.edit(view=self) - except ValueError as e: + except BadArgument as e: await interaction.message.reply( - embed=Embed(colour=discord.Colour.red(), title="Bad Content", description=str(e)) + embed=Embed(colour=discord.Colour.red(), title="Bad Argument", description=str(e)) ) await interaction.message.edit(view=self) else: diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 6bc6659739..eb1615b28c 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -7,6 +7,7 @@ from typing import Literal, Optional, get_type_hints import discord +from botcore.site_api import ResponseCodeError from discord import Colour, Embed, HTTPException, Message from discord.ext import commands from discord.ext.commands import BadArgument, Cog, Context, has_any_role @@ -25,7 +26,7 @@ build_filter_repr_dict, description_and_settings_converter, filter_overrides, populate_embed_from_dict ) from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter -from bot.exts.filtering._ui.ui import ArgumentCompletionView, DeleteConfirmationView +from bot.exts.filtering._ui.ui import ArgumentCompletionView, DeleteConfirmationView, format_response_error from bot.exts.filtering._utils import past_tense, starting_value, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator @@ -403,9 +404,12 @@ async def f_edit( patch_func = partial(self._patch_filter, filter_) if noui: - await patch_func( - ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings - ) + try: + await patch_func( + ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings + ) + except ResponseCodeError as e: + await ctx.reply(embed=format_response_error(e)) return embed = Embed(colour=Colour.blue()) @@ -578,7 +582,11 @@ async def fl_edit( list_type, filter_list = result settings = settings_converter(self.loaded_settings, settings) if noui: - await self._patch_filter_list(ctx.message, filter_list, list_type, settings) + try: + await self._patch_filter_list(ctx.message, filter_list, list_type, settings) + except ResponseCodeError as e: + await ctx.reply(embed=format_response_error(e)) + return embed = Embed(colour=Colour.blue()) embed.set_author(name=f"{filter_list[list_type].label.title()} Filter List") @@ -782,6 +790,8 @@ async def _add_filter( await self._post_new_filter( ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings ) + except ResponseCodeError as e: + await ctx.reply(embed=format_response_error(e)) except ValueError as e: raise BadArgument(str(e)) return From 6b89c1f6040250753262bc3ba878b8f1d1af8598 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 11 Oct 2022 18:29:41 +0300 Subject: [PATCH 049/132] Add filter matching command --- bot/exts/filtering/_filter_context.py | 4 +- bot/exts/filtering/_filter_lists/domain.py | 18 ++++----- bot/exts/filtering/_filter_lists/extension.py | 19 +++++---- .../filtering/_filter_lists/filter_list.py | 10 ++--- bot/exts/filtering/_filter_lists/invite.py | 17 ++++---- bot/exts/filtering/_filter_lists/token.py | 18 ++++----- .../actions/infraction_and_notification.py | 5 ++- bot/exts/filtering/_ui/filter.py | 4 +- bot/exts/filtering/filtering.py | 39 +++++++++++++++++-- 9 files changed, 83 insertions(+), 51 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 02738d452d..5e2f5b45b2 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -4,7 +4,7 @@ from enum import Enum, auto from typing import Optional, Union -from discord import DMChannel, Message, TextChannel, Thread, User +from discord import DMChannel, Member, Message, TextChannel, Thread, User class Event(Enum): @@ -20,7 +20,7 @@ class FilterContext: # Input context event: Event # The type of event - author: User # Who triggered the event + author: User | Member | None # Who triggered the event channel: Union[TextChannel, Thread, DMChannel] # The channel involved content: Union[str, set] # What actually needs filtering message: Optional[Message] # The message involved diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index ec43e92dff..34ab5670c5 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -4,7 +4,6 @@ import typing from functools import reduce from operator import or_ -from typing import Optional, Type from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType @@ -36,20 +35,20 @@ def __init__(self, filtering_cog: Filtering): super().__init__() filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) - def get_filter_type(self, content: str) -> Type[Filter]: + def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" return DomainFilter @property - def filter_types(self) -> set[Type[Filter]]: + def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {DomainFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: - """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" text = ctx.content if not text: - return None, "" + return None, [] text = clean_input(text) urls = {match.group(1).lower().rstrip("/") for match in URL_RE.finditer(text)} @@ -60,7 +59,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings ) ctx.notification_domain = new_ctx.notification_domain actions = None - message = "" + messages = [] if triggers: action_defaults = self[ListType.DENY].defaults.actions actions = reduce( @@ -73,6 +72,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings message = f"#{triggers[0].id} (`{triggers[0].content}`)" if triggers[0].description: message += f" - {triggers[0].description}" + messages = [message] else: - message = ", ".join(f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers) - return actions, message + messages = [f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers] + return actions, messages diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index ce1a46e4aa..a58c6c45e6 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -2,7 +2,6 @@ import typing from os.path import splitext -from typing import Optional, Type import bot from bot.constants import Channels, URLs @@ -53,24 +52,24 @@ def __init__(self, filtering_cog: Filtering): filtering_cog.subscribe(self, Event.MESSAGE) self._whitelisted_description = None - def get_filter_type(self, content: str) -> Type[Filter]: + def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" return ExtensionFilter @property - def filter_types(self) -> set[Type[Filter]]: + def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {ExtensionFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: - """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" # Return early if the message doesn't have attachments. - if not ctx.message.attachments: - return None, "" + if not ctx.message or not ctx.message.attachments: + return None, [] _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) if failed: # There's no extension filtering in this context. - return None, "" + return None, [] # Find all extensions in the message. all_ext = { @@ -84,7 +83,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings not_allowed = {ext: filename for ext, filename in all_ext if ext not in allowed_ext} if not not_allowed: # Yes, it's a double negative. Meaning all attachments are allowed :) - return None, "" + return None, [] # Something is disallowed. if ".py" in not_allowed: @@ -110,4 +109,4 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings ) ctx.matches += not_allowed.values() - return self[ListType.ALLOW].defaults.actions, ", ".join(f"`{ext}`" for ext in not_allowed) + return self[ListType.ALLOW].defaults.actions, [f"`{ext}`" for ext in not_allowed] diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index ecbcb8f091..daab45b811 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,7 +1,7 @@ from abc import abstractmethod from collections.abc import Iterator from enum import Enum -from typing import Any, ItemsView, NamedTuple, Optional, Type +from typing import Any, ItemsView, NamedTuple from discord.ext.commands import BadArgument @@ -128,17 +128,17 @@ def add_filter(self, list_type: ListType, filter_data: dict) -> Filter: return new_filter @abstractmethod - def get_filter_type(self, content: str) -> Type[Filter]: + def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" @property @abstractmethod - def filter_types(self) -> set[Type[Filter]]: + def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" @abstractmethod - async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: - """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" @staticmethod def filter_list_result( diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 30884a2ab9..5bb4549ae6 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -3,7 +3,6 @@ import typing from functools import reduce from operator import or_ -from typing import Optional, Type from botcore.utils.regex import DISCORD_INVITE from discord import Embed, Invite @@ -42,20 +41,20 @@ def __init__(self, filtering_cog: Filtering): super().__init__() filtering_cog.subscribe(self, Event.MESSAGE) - def get_filter_type(self, content: str) -> Type[Filter]: + def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" return InviteFilter @property - def filter_types(self) -> set[Type[Filter]]: + def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {InviteFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: - """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) if failed: # There's no invite filtering in this context. - return None, "" + return None, [] text = clean_input(ctx.content) @@ -65,7 +64,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings matches = list(DISCORD_INVITE.finditer(text)) invite_codes = {m.group("invite") for m in matches} if not invite_codes: - return None, "" + return None, [] # Sort the invites into three categories: denied_by_default = dict() # Denied unless whitelisted. @@ -107,7 +106,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings }) if not disallowed_invites: - return None, "" + return None, [] actions = None if len(disallowed_invites) > len(triggered): # There are invites which weren't allowed but aren't blacklisted. @@ -124,7 +123,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings actions = reduce(or_, (filter_.actions for filter_ in triggered)) ctx.matches += {match[0] for match in matches if match.group("invite") in disallowed_invites} ctx.alert_embeds += (self._guild_embed(invite) for invite in disallowed_invites.values() if invite) - return actions, ", ".join(f"`{invite}`" for invite in disallowed_invites) + return actions, [f"`{invite}`" for invite in disallowed_invites] @staticmethod def _guild_embed(invite: Invite) -> Embed: diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index 2abf945534..c80ccfd68f 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -4,7 +4,6 @@ import typing from functools import reduce from operator import or_ -from typing import Optional, Type from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType @@ -37,20 +36,20 @@ def __init__(self, filtering_cog: Filtering): super().__init__() filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) - def get_filter_type(self, content: str) -> Type[Filter]: + def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" return TokenFilter @property - def filter_types(self) -> set[Type[Filter]]: + def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {TokenFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], Optional[str]]: - """Dispatch the given event to the list's filters, and return actions to take and a message to relay to mods.""" + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" text = ctx.content if not text: - return None, "" + return None, [] if SPOILER_RE.search(text): text = self._expand_spoilers(text) text = clean_input(text) @@ -60,7 +59,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings ctx, self[ListType.DENY].filters, self[ListType.DENY].defaults.validations ) actions = None - message = "" + messages = [] if triggers: action_defaults = self[ListType.DENY].defaults.actions actions = reduce( @@ -73,9 +72,10 @@ async def actions_for(self, ctx: FilterContext) -> tuple[Optional[ActionSettings message = f"#{triggers[0].id} (`{triggers[0].content}`)" if triggers[0].description: message += f" - {triggers[0].description}" + messages = [message] else: - message = ", ".join(f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers) - return actions, message + messages = [f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers] + return actions, messages @staticmethod def _expand_spoilers(text: str) -> str: diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 4ec06ef4c6..7835a7d0bd 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -106,7 +106,10 @@ class InfractionAndNotification(ActionEntry): ) + ", ".join(infraction.name for infraction in Infraction), "infraction_duration": "How long the infraction should last for in seconds, or 'None' for permanent.", "infraction_reason": "The reason delivered with the infraction.", - "infraction_channel": "The channel ID in which to invoke the infraction (and send the confirmation message).", + "infraction_channel": ( + "The channel ID in which to invoke the infraction (and send the confirmation message). " + "If blank, the infraction will be sent in the context channel." + ), "dm_content": "The contents of a message to be DMed to the offending user.", "dm_embed": "The contents of the embed to be DMed to the offending user." } diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index e6a568ad0b..e6330329d2 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -452,9 +452,9 @@ def template_settings(filter_id: str, filter_list: FilterList, list_type: ListTy except ValueError: raise ValueError("Template value must be a non-negative integer.") - if filter_id not in filter_list.filter_lists[list_type]: + if filter_id not in filter_list[list_type].filters: raise ValueError( f"Could not find filter with ID `{filter_id}` in the {list_type.name} {filter_list.name} list." ) - filter_ = filter_list.filter_lists[list_type][filter_id] + filter_ = filter_list[list_type].filters[filter_id] return filter_overrides(filter_, filter_list, list_type) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index eb1615b28c..c47ba653f0 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -16,7 +16,7 @@ import bot import bot.exts.filtering._ui.filter as filters_ui from bot.bot import Bot -from bot.constants import Colours, MODERATION_ROLES, Roles, Webhooks +from bot.constants import Channels, Colours, MODERATION_ROLES, Roles, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filter_lists.filter_list import AtomicList @@ -492,6 +492,37 @@ async def s_describe(self, ctx: Context, setting_name: Optional[str]) -> None: embed.colour = Colour.blue() await ctx.send(embed=embed) + @filter.command(name="match") + async def f_match(self, ctx: Context, message: Message | None, *, string: str | None) -> None: + """ + Post any responses from the filter lists for the given message or string. + + If there's a message the string will be ignored. Note that if a message is provided, it will go through all + validations appropriate to where it was sent and who sent it. + + If a string is provided, it will be validated in the context of a user with no roles in python-general. + """ + if not message and not string: + raise BadArgument(":x: Please provide input.") + if message: + filter_ctx = FilterContext( + Event.MESSAGE, message.author, message.channel, message.content, message, message.embeds + ) + else: + filter_ctx = FilterContext( + Event.MESSAGE, None, ctx.guild.get_channel(Channels.python_general), string, None + ) + + _, list_messages = await self._resolve_action(filter_ctx) + lines = [] + for filter_list, list_message_list in list_messages.items(): + if list_message_list: + lines.extend([f"**{filter_list.name.title()}s**", *list_message_list, "\n"]) + lines = lines[:-1] # Remove last newline. + + embed = Embed(colour=Colour.blue(), title="Match results") + await LinePaginator.paginate(lines, ctx, embed, max_lines=10, empty=False) + # endregion # region: filterlist group @@ -651,7 +682,7 @@ def _load_raw_filter_list(self, list_data: dict) -> AtomicList | None: self.filter_lists[list_name] = filter_list_types[list_name](self) return self.filter_lists[list_name].add_list(list_data) - async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], dict[FilterList, str]]: + async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], dict[FilterList, list[str]]]: """ Return the actions that should be taken for all filter lists in the given context. @@ -673,7 +704,7 @@ async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSett return result_actions, messages - async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, str]) -> None: + async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, list[str]]) -> None: """Build an alert message from the filter context, and send it via the alert webhook.""" if not self.webhook: return @@ -691,7 +722,7 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi filters = [] for filter_list, list_message in triggered_filters.items(): if list_message: - filters.append(f"**{filter_list.name.title()} Filters:** {list_message}") + filters.append(f"**{filter_list.name.title()} Filters:** {', '.join(list_message)}") filters = "\n".join(filters) matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) From 2b8f85b98a2d4f372f2767ae314450db7c04c393 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 15 Oct 2022 14:50:13 +0300 Subject: [PATCH 050/132] More improvements to FilterList Methods that work with only a specific list type are moved to AtomicList. FilterList is now a subclass of dict, since that's what it is. --- bot/exts/filtering/_filter_lists/domain.py | 4 +- .../filtering/_filter_lists/filter_list.py | 119 +++++++----------- bot/exts/filtering/_filter_lists/invite.py | 4 +- bot/exts/filtering/_filter_lists/token.py | 4 +- bot/exts/filtering/_ui/filter.py | 6 +- bot/exts/filtering/_ui/filter_list.py | 2 +- bot/exts/filtering/filtering.py | 4 +- 7 files changed, 54 insertions(+), 89 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index 34ab5670c5..17984e276d 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -54,9 +54,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, urls = {match.group(1).lower().rstrip("/") for match in URL_RE.finditer(text)} new_ctx = ctx.replace(content=urls) - triggers = self.filter_list_result( - new_ctx, self[ListType.DENY].filters, self[ListType.DENY].defaults.validations - ) + triggers = self[ListType.DENY].filter_list_result(new_ctx) ctx.notification_domain = new_ctx.notification_domain actions = None messages = [] diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index daab45b811..9eb907fc1b 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,7 +1,6 @@ from abc import abstractmethod -from collections.abc import Iterator from enum import Enum -from typing import Any, ItemsView, NamedTuple +from typing import Any, NamedTuple from discord.ext.commands import BadArgument @@ -62,35 +61,53 @@ def label(self) -> str: """Provide a short description identifying the list with its name and type.""" return f"{past_tense(self.list_type.name.lower())} {self.name.lower()}" + def filter_list_result(self, ctx: FilterContext) -> list[Filter]: + """ + Sift through the list of filters, and return only the ones which apply to the given context. -class FilterList(FieldRequiring): - """Dispatches events to lists of _filters, and aggregates the responses into a single list of actions to take.""" - - # Each subclass must define a name matching the filter_list name we're expecting to receive from the database. - # Names must be unique across all filter lists. - name = FieldRequiring.MUST_SET_UNIQUE + The strategy is as follows: + 1. The default settings are evaluated on the given context. The default answer for whether the filter is + relevant in the given context is whether there aren't any validation settings which returned False. + 2. For each filter, its overrides are considered: + - If there are no overrides, then the filter is relevant if that is the default answer. + - Otherwise it is relevant if there are no failed overrides, and any failing default is overridden by a + successful override. - def __init__(self): - self._filter_lists: dict[ListType, AtomicList] = {} + If the filter is relevant in context, see if it actually triggers. + """ + passed_by_default, failed_by_default = self.defaults.validations.evaluate(ctx) + default_answer = not bool(failed_by_default) - def __iter__(self) -> Iterator[ListType]: - return iter(self._filter_lists) + relevant_filters = [] + for filter_ in self.filters.values(): + if not filter_.validations: + if default_answer and filter_.triggered_on(ctx): + relevant_filters.append(filter_) + else: + passed, failed = filter_.validations.evaluate(ctx) + if not failed and failed_by_default < passed: + if filter_.triggered_on(ctx): + relevant_filters.append(filter_) - def __getitem__(self, list_type: ListType) -> AtomicList: - return self._filter_lists[list_type] + return relevant_filters - def __contains__(self, list_type: ListType) -> bool: - return list_type in self._filter_lists + def default(self, setting: str) -> Any: + """Get the default value of a specific setting.""" + missing = object() + value = self.defaults.actions.get_setting(setting, missing) + if value is missing: + value = self.defaults.validations.get_setting(setting, missing) + if value is missing: + raise ValueError(f"Couldn't find a setting named {setting!r}.") + return value - def __bool__(self) -> bool: - return bool(self._filter_lists) - def __len__(self) -> int: - return len(self._filter_lists) +class FilterList(FieldRequiring, dict[ListType, AtomicList]): + """Dispatches events to lists of _filters, and aggregates the responses into a single list of actions to take.""" - def items(self) -> ItemsView[ListType, AtomicList]: - """Return an iterator for the lists' types and values.""" - return self._filter_lists.items() + # Each subclass must define a name matching the filter_list name we're expecting to receive from the database. + # Names must be unique across all filter lists. + name = FieldRequiring.MUST_SET_UNIQUE def add_list(self, list_data: dict) -> AtomicList: """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" @@ -102,24 +119,8 @@ def add_list(self, list_data: dict) -> AtomicList: for filter_data in list_data["filters"]: filters[filter_data["id"]] = self._create_filter(filter_data) - self._filter_lists[list_type] = AtomicList(list_data["id"], self.name, list_type, defaults, filters) - return self._filter_lists[list_type] - - def remove_list(self, list_type: ListType) -> None: - """Remove the list associated with the given type from the FilterList object.""" - if list_type not in self._filter_lists: - return - self._filter_lists.pop(list_type) - - def default(self, list_type: ListType, setting: str) -> Any: - """Get the default value of a specific setting.""" - missing = object() - value = self._filter_lists[list_type].defaults.actions.get_setting(setting, missing) - if value is missing: - value = self._filter_lists[list_type].defaults.validations.get_setting(setting, missing) - if value is missing: - raise ValueError(f"Could find a setting named {setting}.") - return value + self[list_type] = AtomicList(list_data["id"], self.name, list_type, defaults, filters) + return self[list_type] def add_filter(self, list_type: ListType, filter_data: dict) -> Filter: """Add a filter to the list of the specified type.""" @@ -140,39 +141,6 @@ def filter_types(self) -> set[type[Filter]]: async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" - @staticmethod - def filter_list_result( - ctx: FilterContext, filters: dict[int, Filter], defaults: ValidationSettings - ) -> list[Filter]: - """ - Sift through the list of filters, and return only the ones which apply to the given context. - - The strategy is as follows: - 1. The default settings are evaluated on the given context. The default answer for whether the filter is - relevant in the given context is whether there aren't any validation settings which returned False. - 2. For each filter, its overrides are considered: - - If there are no overrides, then the filter is relevant if that is the default answer. - - Otherwise it is relevant if there are no failed overrides, and any failing default is overridden by a - successful override. - - If the filter is relevant in context, see if it actually triggers. - """ - passed_by_default, failed_by_default = defaults.evaluate(ctx) - default_answer = not bool(failed_by_default) - - relevant_filters = [] - for filter_ in filters.values(): - if not filter_.validations: - if default_answer and filter_.triggered_on(ctx): - relevant_filters.append(filter_) - else: - passed, failed = filter_.validations.evaluate(ctx) - if not failed and failed_by_default < passed: - if filter_.triggered_on(ctx): - relevant_filters.append(filter_) - - return relevant_filters - def _create_filter(self, filter_data: dict) -> Filter: """Create a filter from the given data.""" try: @@ -182,3 +150,6 @@ def _create_filter(self, filter_data: dict) -> Filter: log.warning(e) else: return new_filter + + def __hash__(self): + return hash(id(self)) diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 5bb4549ae6..d35fdd4a43 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -97,9 +97,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, # Add the allowed by default only if they're blacklisted. guilds_for_inspection = {invite.guild.id for invite in allowed_by_default.values()} new_ctx = ctx.replace(content=guilds_for_inspection) - triggered = self.filter_list_result( - new_ctx, self[ListType.ALLOW].filters, self[ListType.DENY].defaults.validations - ) + triggered = self[ListType.ALLOW].filter_list_result(new_ctx) disallowed_invites.update({ invite_code: invite for invite_code, invite in allowed_by_default.items() if invite.guild.id in {filter_.content for filter_ in triggered} diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index c80ccfd68f..4b161d9b7d 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -55,9 +55,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, text = clean_input(text) ctx = ctx.replace(content=text) - triggers = self.filter_list_result( - ctx, self[ListType.DENY].filters, self[ListType.DENY].defaults.validations - ) + triggers = self[ListType.DENY].filter_list_result(ctx) actions = None messages = [] if triggers: diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index e6330329d2..38eef3ca67 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -278,7 +278,7 @@ async def update_embed( default_value = self.filter_type.extra_fields_type().dict()[setting_name] else: dict_to_edit = self.settings_overrides - default_value = self.filter_list.default(self.list_type, setting_name) + default_value = self.filter_list[self.list_type].default(setting_name) # Update the setting override value or remove it if setting_value is not self._REMOVE: if not repr_equals(setting_value, default_value): @@ -405,7 +405,7 @@ def description_and_settings_converter( type_ = loaded_settings[setting][2] try: parsed_value = parse_value(settings.pop(setting), type_) - if not repr_equals(parsed_value, filter_list.default(list_type, setting)): + if not repr_equals(parsed_value, filter_list[list_type].default(setting)): settings[setting] = parsed_value except (TypeError, ValueError) as e: raise BadArgument(e) @@ -431,7 +431,7 @@ def filter_overrides(filter_: Filter, filter_list: FilterList, list_type: ListTy if settings: for _, setting in settings.items(): for setting_name, value in to_serializable(setting.dict()).items(): - if not repr_equals(value, filter_list.default(list_type, setting_name)): + if not repr_equals(value, filter_list[list_type].default(setting_name)): overrides_values[setting_name] = value if filter_.extra_fields_type: diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index b072e293ed..15d81322b2 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -234,7 +234,7 @@ async def update_embed( if not setting_name: # Obligatory check to match the signature in the parent class. return - default_value = self.filter_list.default(self.list_type, setting_name) + default_value = self.filter_list[self.list_type].default(setting_name) if not repr_equals(setting_value, default_value): self.settings[setting_name] = setting_value # If there's already a new value, remove it, since the new value is the same as the default. diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index c47ba653f0..563bdacb59 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -646,7 +646,7 @@ async def delete_list() -> None: file = discord.File(BytesIO(json.dumps(list_data, indent=4).encode("utf-8")), f"{list_description}.json") message = await ctx.send("⏳ Annihilation in progress, please hold...", file=file) # Unload the filter list. - filter_list.remove_list(list_type) + filter_list.pop(list_type) if not filter_list: # There's nothing left, remove from the cog. self.filter_lists.pop(filter_list.name) self.unsubscribe(filter_list) @@ -950,7 +950,7 @@ async def _patch_filter_list(msg: Message, filter_list: FilterList, list_type: L response = await bot.instance.api_client.patch( f'bot/filter/filter_lists/{list_id}', json=to_serializable(settings) ) - filter_list.remove_list(list_type) + filter_list.pop(list_type, None) filter_list.add_list(response) await msg.reply(f"✅ Edited filter list: {filter_list[list_type].label}") From e7e65c0056003557b2307ce4f5167ca076c4154f Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 16 Oct 2022 20:55:28 +0300 Subject: [PATCH 051/132] Change override handling The Settings class is now a subclass of dict as well. SettingsEntry's now store which fields are overrides in case the entry is not empty. If there is a overridden value and a non-overridden value in the same entry, the non-overridden value will now correctly be the default instead of staying empty. Additionally fixes bugs with free list input parsing. --- .../filtering/_filter_lists/filter_list.py | 17 +-- bot/exts/filtering/_filters/filter.py | 28 ++++- bot/exts/filtering/_filters/invite.py | 4 +- bot/exts/filtering/_settings.py | 109 ++++++++++-------- .../filtering/_settings_types/actions/ping.py | 15 ++- .../_settings_types/settings_entry.py | 31 ++++- bot/exts/filtering/_ui/filter.py | 23 +--- bot/exts/filtering/_ui/ui.py | 24 ++-- bot/exts/filtering/filtering.py | 6 +- 9 files changed, 143 insertions(+), 114 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 9eb907fc1b..f993665f27 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -6,7 +6,7 @@ from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filters.filter import Filter -from bot.exts.filtering._settings import ActionSettings, ValidationSettings, create_settings +from bot.exts.filtering._settings import ActionSettings, Defaults, create_settings from bot.exts.filtering._utils import FieldRequiring, past_tense from bot.log import get_logger @@ -36,13 +36,6 @@ def list_type_converter(argument: str) -> ListType: raise BadArgument(f"No matching list type found for {argument!r}.") -class Defaults(NamedTuple): - """Represents an atomic list's default settings.""" - - actions: ActionSettings - validations: ValidationSettings - - class AtomicList(NamedTuple): """ Represents the atomic structure of a single filter list as it appears in the database. @@ -117,14 +110,14 @@ def add_list(self, list_data: dict) -> AtomicList: filters = {} for filter_data in list_data["filters"]: - filters[filter_data["id"]] = self._create_filter(filter_data) + filters[filter_data["id"]] = self._create_filter(filter_data, defaults) self[list_type] = AtomicList(list_data["id"], self.name, list_type, defaults, filters) return self[list_type] def add_filter(self, list_type: ListType, filter_data: dict) -> Filter: """Add a filter to the list of the specified type.""" - new_filter = self._create_filter(filter_data) + new_filter = self._create_filter(filter_data, self[list_type].defaults) self[list_type].filters[filter_data["id"]] = new_filter return new_filter @@ -141,11 +134,11 @@ def filter_types(self) -> set[type[Filter]]: async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" - def _create_filter(self, filter_data: dict) -> Filter: + def _create_filter(self, filter_data: dict, defaults: Defaults) -> Filter: """Create a filter from the given data.""" try: filter_type = self.get_filter_type(filter_data["content"]) - new_filter = filter_type(filter_data) + new_filter = filter_type(filter_data, defaults) except TypeError as e: log.warning(e) else: diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index b4a2bfe5e2..0d11d5b3cb 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,10 +1,10 @@ from abc import abstractmethod -from typing import Optional +from typing import Any, Optional from pydantic import ValidationError from bot.exts.filtering._filter_context import FilterContext -from bot.exts.filtering._settings import create_settings +from bot.exts.filtering._settings import Defaults, create_settings from bot.exts.filtering._utils import FieldRequiring @@ -22,14 +22,30 @@ class Filter(FieldRequiring): # If a subclass uses extra fields, it should assign the pydantic model type to this variable. extra_fields_type = None - def __init__(self, filter_data: dict): + def __init__(self, filter_data: dict, defaults: Defaults | None = None): self.id = filter_data["id"] self.content = filter_data["content"] self.description = filter_data["description"] - self.actions, self.validations = create_settings(filter_data["settings"]) - self.extra_fields = filter_data["additional_field"] or "{}" # noqa: P103 + self.actions, self.validations = create_settings(filter_data["settings"], defaults=defaults) if self.extra_fields_type: - self.extra_fields = self.extra_fields_type.parse_raw(self.extra_fields) + self.extra_fields = self.extra_fields_type.parse_raw(filter_data["additional_field"]) + else: + self.extra_fields = None + + @property + def overrides(self) -> tuple[dict[str, Any], dict[str, Any]]: + """Return a tuple of setting overrides and filter setting overrides.""" + settings = {} + if self.actions: + settings = self.actions.overrides + if self.validations: + settings |= self.validations.overrides + + filter_settings = {} + if self.extra_fields: + filter_settings = self.extra_fields.dict(exclude_unset=True) + + return settings, filter_settings @abstractmethod def triggered_on(self, ctx: FilterContext) -> bool: diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py index 0463b0032a..ac4f62cb6d 100644 --- a/bot/exts/filtering/_filters/invite.py +++ b/bot/exts/filtering/_filters/invite.py @@ -16,8 +16,8 @@ class InviteFilter(Filter): name = "invite" - def __init__(self, filter_data: dict): - super().__init__(filter_data) + def __init__(self, filter_data: dict, defaults_data: dict | None = None): + super().__init__(filter_data, defaults_data) self.content = int(self.content) def triggered_on(self, ctx: FilterContext) -> bool: diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index 7b09e3c520..4c2114f07f 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -1,11 +1,13 @@ from __future__ import annotations +import operator from abc import abstractmethod -from typing import Any, Iterator, Mapping, Optional, TypeVar +from functools import reduce +from typing import Any, NamedTuple, Optional, TypeVar from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types import settings_types -from bot.exts.filtering._settings_types.settings_entry import ActionEntry, ValidationEntry +from bot.exts.filtering._settings_types.settings_entry import ActionEntry, SettingsEntry, ValidationEntry from bot.exts.filtering._utils import FieldRequiring from bot.log import get_logger @@ -15,14 +17,18 @@ _already_warned: set[str] = set() +T = TypeVar("T", bound=SettingsEntry) + def create_settings( - settings_data: dict, *, keep_empty: bool = False + settings_data: dict, *, defaults: Defaults | None = None, keep_empty: bool = False ) -> tuple[Optional[ActionSettings], Optional[ValidationSettings]]: """ Create and return instances of the Settings subclasses from the given data. Additionally, warn for data entries with no matching class. + + In case these are setting overrides, the defaults can be provided to keep track of the correct values. """ action_data = {} validation_data = {} @@ -36,13 +42,18 @@ def create_settings( f"A setting named {entry_name} was loaded from the database, but no matching class." ) _already_warned.add(entry_name) + if defaults is None: + default_actions = None + default_validations = None + else: + default_actions, default_validations = defaults return ( - ActionSettings.create(action_data, keep_empty=keep_empty), - ValidationSettings.create(validation_data, keep_empty=keep_empty) + ActionSettings.create(action_data, defaults=default_actions, keep_empty=keep_empty), + ValidationSettings.create(validation_data, defaults=default_validations, keep_empty=keep_empty) ) -class Settings(FieldRequiring): +class Settings(FieldRequiring, dict[str, T]): """ A collection of settings. @@ -54,13 +65,13 @@ class Settings(FieldRequiring): the filter list which contains the filter. """ - entry_type = FieldRequiring.MUST_SET + entry_type = T _already_warned: set[str] = set() @abstractmethod - def __init__(self, settings_data: dict, *, keep_empty: bool = False): - self._entries: dict[str, Settings.entry_type] = {} + def __init__(self, settings_data: dict, *, defaults: Settings | None = None, keep_empty: bool = False): + super().__init__() entry_classes = settings_types.get(self.entry_type.__name__) for entry_name, entry_data in settings_data.items(): @@ -75,63 +86,55 @@ def __init__(self, settings_data: dict, *, keep_empty: bool = False): self._already_warned.add(entry_name) else: try: - new_entry = entry_cls.create(entry_data, keep_empty=keep_empty) + entry_defaults = None if defaults is None else defaults[entry_name] + new_entry = entry_cls.create( + entry_data, defaults=entry_defaults, keep_empty=keep_empty + ) if new_entry: - self._entries[entry_name] = new_entry + self[entry_name] = new_entry except TypeError as e: raise TypeError( f"Attempted to load a {entry_name} setting, but the response is malformed: {entry_data}" ) from e - def __contains__(self, item: str) -> bool: - return item in self._entries - - def __setitem__(self, key: str, value: entry_type) -> None: - self._entries[key] = value + @property + def overrides(self) -> dict[str, Any]: + """Return a dictionary of overrides across all entries.""" + return reduce(operator.or_, (entry.overrides for entry in self.values() if entry), {}) def copy(self: TSettings) -> TSettings: """Create a shallow copy of the object.""" copy = self.__class__({}) - copy._entries = self._entries.copy() + copy.update(super().copy()) # Copy the internal dict. return copy - def items(self) -> Iterator[tuple[str, entry_type]]: - """Return an iterator for the items in the entries dictionary.""" - yield from self._entries.items() - - def update(self, mapping: Mapping[str, entry_type], **kwargs: entry_type) -> None: - """Update the entries with items from `mapping` and the kwargs.""" - self._entries.update(mapping, **kwargs) - - def get(self, key: str, default: Optional[Any] = None) -> entry_type: - """Get the entry matching the key, or fall back to the default value if the key is missing.""" - return self._entries.get(key, default) - def get_setting(self, key: str, default: Optional[Any] = None) -> Any: """Get the setting matching the key, or fall back to the default value if the key is missing.""" - for entry in self._entries.values(): + for entry in self.values(): if hasattr(entry, key): return getattr(entry, key) return default @classmethod - def create(cls, settings_data: dict, *, keep_empty: bool = False) -> Optional[Settings]: + def create( + cls, settings_data: dict, *, defaults: Settings | None = None, keep_empty: bool = False + ) -> Optional[Settings]: """ Returns a Settings object from `settings_data` if it holds any value, None otherwise. Use this method to create Settings objects instead of the init. The None value is significant for how a filter list iterates over its filters. """ - settings = cls(settings_data, keep_empty=keep_empty) + settings = cls(settings_data, defaults=defaults, keep_empty=keep_empty) # If an entry doesn't hold any values, its `create` method will return None. # If all entries are None, then the settings object holds no values. - if not keep_empty and not any(settings._entries.values()): + if not keep_empty and not any(settings.values()): return None return settings -class ValidationSettings(Settings): +class ValidationSettings(Settings[ValidationEntry]): """ A collection of validation settings. @@ -141,16 +144,15 @@ class ValidationSettings(Settings): entry_type = ValidationEntry - def __init__(self, settings_data: dict, *, keep_empty: bool = False): - super().__init__(settings_data, keep_empty=keep_empty) + def __init__(self, settings_data: dict, *, defaults: Settings | None = None, keep_empty: bool = False): + super().__init__(settings_data, defaults=defaults, keep_empty=keep_empty) def evaluate(self, ctx: FilterContext) -> tuple[set[str], set[str]]: """Evaluates for each setting whether the context is relevant to the filter.""" passed = set() failed = set() - self._entries: dict[str, ValidationEntry] - for name, validation in self._entries.items(): + for name, validation in self.items(): if validation: if validation.triggers_on(ctx): passed.add(name) @@ -160,7 +162,7 @@ def evaluate(self, ctx: FilterContext) -> tuple[set[str], set[str]]: return passed, failed -class ActionSettings(Settings): +class ActionSettings(Settings[ActionEntry]): """ A collection of action settings. @@ -170,21 +172,21 @@ class ActionSettings(Settings): entry_type = ActionEntry - def __init__(self, settings_data: dict, *, keep_empty: bool = False): - super().__init__(settings_data, keep_empty=keep_empty) + def __init__(self, settings_data: dict, *, defaults: Settings | None = None, keep_empty: bool = False): + super().__init__(settings_data, defaults=defaults, keep_empty=keep_empty) def __or__(self, other: ActionSettings) -> ActionSettings: """Combine the entries of two collections of settings into a new ActionsSettings.""" actions = {} # A settings object doesn't necessarily have all types of entries (e.g in the case of filter overrides). - for entry in self._entries: - if entry in other._entries: - actions[entry] = self._entries[entry] | other._entries[entry] + for entry in self: + if entry in other: + actions[entry] = self[entry] | other[entry] else: - actions[entry] = self._entries[entry] - for entry in other._entries: + actions[entry] = self[entry] + for entry in other: if entry not in actions: - actions[entry] = other._entries[entry] + actions[entry] = other[entry] result = ActionSettings({}) result.update(actions) @@ -192,13 +194,20 @@ def __or__(self, other: ActionSettings) -> ActionSettings: async def action(self, ctx: FilterContext) -> None: """Execute the action of every action entry stored.""" - for entry in self._entries.values(): + for entry in self.values(): await entry.action(ctx) def fallback_to(self, fallback: ActionSettings) -> ActionSettings: """Fill in missing entries from `fallback`.""" new_actions = self.copy() for entry_name, entry_value in fallback.items(): - if entry_name not in self._entries: - new_actions._entries[entry_name] = entry_value + if entry_name not in self: + new_actions[entry_name] = entry_value return new_actions + + +class Defaults(NamedTuple): + """Represents an atomic list's default settings.""" + + actions: ActionSettings + validations: ValidationSettings diff --git a/bot/exts/filtering/_settings_types/actions/ping.py b/bot/exts/filtering/_settings_types/actions/ping.py index 85590478cb..faac8f4b9c 100644 --- a/bot/exts/filtering/_settings_types/actions/ping.py +++ b/bot/exts/filtering/_settings_types/actions/ping.py @@ -1,9 +1,10 @@ from functools import cache from typing import ClassVar -from discord import Guild from pydantic import validator +import bot +from bot.constants import Guild from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry @@ -37,7 +38,7 @@ def init_sequence_if_none(cls, pings: list[str] | None) -> list[str]: async def action(self, ctx: FilterContext) -> None: """Add the stored pings to the alert message content.""" mentions = self.guild_pings if ctx.channel.guild else self.dm_pings - new_content = " ".join([self._resolve_mention(mention, ctx.channel.guild) for mention in mentions]) + new_content = " ".join([self._resolve_mention(mention) for mention in mentions]) ctx.alert_content = f"{new_content} {ctx.alert_content}" def __or__(self, other: ActionEntry): @@ -49,12 +50,16 @@ def __or__(self, other: ActionEntry): @staticmethod @cache - def _resolve_mention(mention: str, guild: Guild) -> str: + def _resolve_mention(mention: str) -> str: """Return the appropriate formatting for the formatting, be it a literal, a user ID, or a role ID.""" + guild = bot.instance.get_guild(Guild.id) if mention in ("here", "everyone"): return f"@{mention}" - if mention.isdigit(): # It's an ID. - mention = int(mention) + try: + mention = int(mention) # It's an ID. + except ValueError: + pass + else: if any(mention == role.id for role in guild.roles): return f"<@&{mention}>" else: diff --git a/bot/exts/filtering/_settings_types/settings_entry.py b/bot/exts/filtering/_settings_types/settings_entry.py index 5a7e41cac1..31e11108de 100644 --- a/bot/exts/filtering/_settings_types/settings_entry.py +++ b/bot/exts/filtering/_settings_types/settings_entry.py @@ -1,9 +1,9 @@ from __future__ import annotations from abc import abstractmethod -from typing import Any, ClassVar, Optional, Union +from typing import Any, ClassVar, Union -from pydantic import BaseModel +from pydantic import BaseModel, PrivateAttr from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._utils import FieldRequiring @@ -19,12 +19,33 @@ class SettingsEntry(BaseModel, FieldRequiring): # Each subclass must define a name matching the entry name we're expecting to receive from the database. # Names must be unique across all filter lists. name: ClassVar[str] = FieldRequiring.MUST_SET_UNIQUE - # Each subclass must define a description of what it does. If the data an entry type receives is comprised of + # Each subclass must define a description of what it does. If the data an entry type receives comprises # several DB fields, the value should a dictionary of field names and their descriptions. description: ClassVar[Union[str, dict[str, str]]] = FieldRequiring.MUST_SET + _overrides: set[str] = PrivateAttr(default_factory=set) + + def __init__(self, defaults: SettingsEntry | None = None, /, **data): + overrides = set() + if defaults: + defaults_dict = defaults.dict() + for field_name, field_value in list(data.items()): + if field_value is None: + data[field_name] = defaults_dict[field_name] + else: + overrides.add(field_name) + super().__init__(**data) + self._overrides |= overrides + + @property + def overrides(self) -> dict[str, Any]: + """Return a dictionary of overrides.""" + return {name: getattr(self, name) for name in self._overrides} + @classmethod - def create(cls, entry_data: Optional[dict[str, Any]], *, keep_empty: bool = False) -> Optional[SettingsEntry]: + def create( + cls, entry_data: dict[str, Any] | None, *, defaults: SettingsEntry | None = None, keep_empty: bool = False + ) -> SettingsEntry | None: """ Returns a SettingsEntry object from `entry_data` if it holds any value, None otherwise. @@ -38,7 +59,7 @@ def create(cls, entry_data: Optional[dict[str, Any]], *, keep_empty: bool = Fals if not isinstance(entry_data, dict): entry_data = {cls.name: entry_data} - return cls(**entry_data) + return cls(defaults, **entry_data) class ValidationEntry(SettingsEntry): diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index 38eef3ca67..765fba683d 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -424,23 +424,10 @@ def description_and_settings_converter( return description, settings, filter_settings -def filter_overrides(filter_: Filter, filter_list: FilterList, list_type: ListType) -> tuple[dict, dict]: - """Get the filter's overrides to the filter list settings and the extra fields settings.""" - overrides_values = {} - for settings in (filter_.actions, filter_.validations): - if settings: - for _, setting in settings.items(): - for setting_name, value in to_serializable(setting.dict()).items(): - if not repr_equals(value, filter_list[list_type].default(setting_name)): - overrides_values[setting_name] = value - - if filter_.extra_fields_type: - # The values here can be safely used since overrides equal to the defaults won't be saved. - extra_fields_overrides = filter_.extra_fields.dict(exclude_unset=True) - else: - extra_fields_overrides = {} - - return overrides_values, extra_fields_overrides +def filter_serializable_overrides(filter_: Filter) -> tuple[dict, dict]: + """Get a serializable version of the filter's overrides.""" + overrides_values, extra_fields_overrides = filter_.overrides + return to_serializable(overrides_values), to_serializable(extra_fields_overrides) def template_settings(filter_id: str, filter_list: FilterList, list_type: ListType) -> tuple[dict, dict]: @@ -457,4 +444,4 @@ def template_settings(filter_id: str, filter_list: FilterList, list_type: ListTy f"Could not find filter with ID `{filter_id}` in the {list_type.name} {filter_list.name} list." ) filter_ = filter_list[list_type].filters[filter_id] - return filter_overrides(filter_, filter_list, list_type) + return filter_serializable_overrides(filter_) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 5a60bb21e6..980eba02af 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -291,10 +291,8 @@ async def apply_removal(self, interaction: Interaction, select: discord.ui.Selec if _i != len(self.stored_value): self.stored_value.pop(_i) - select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] - if not self.stored_value: - self.remove_item(self.removal_select) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self.copy()) + self.stop() async def apply_addition(self, interaction: Interaction, item: str) -> None: """Add an item to the list.""" @@ -303,18 +301,14 @@ async def apply_addition(self, interaction: Interaction, item: str) -> None: return self.stored_value.append(item) - self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] - if len(self.stored_value) == 1: - self.add_item(self.removal_select) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self.copy()) + self.stop() async def apply_edit(self, interaction: Interaction, new_list: str) -> None: """Change the contents of the list.""" - self.stored_value = list(set(new_list.split(","))) - self.removal_select.options = [SelectOption(label=item) for item in self.stored_value[:MAX_SELECT_ITEMS]] - if len(self.stored_value) == 1: - self.add_item(self.removal_select) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self) + self.stored_value = list(set(part.strip() for part in new_list.split(",") if part.strip())) + await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self.copy()) + self.stop() @discord.ui.button(label="Add Value") async def add_value(self, interaction: Interaction, button: discord.ui.Button) -> None: @@ -340,6 +334,10 @@ async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> N await interaction.response.edit_message(content="🚫 Canceled", view=None) self.stop() + def copy(self) -> SequenceEditView: + """Return a copy of this view.""" + return SequenceEditView(self.setting_name, self.stored_value, self.update_callback) + class EnumSelectView(discord.ui.View): """A view containing an instance of EnumSelect.""" diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 563bdacb59..6ff5181a92 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -23,7 +23,7 @@ from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._ui.filter import ( - build_filter_repr_dict, description_and_settings_converter, filter_overrides, populate_embed_from_dict + build_filter_repr_dict, description_and_settings_converter, filter_serializable_overrides, populate_embed_from_dict ) from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter from bot.exts.filtering._ui.ui import ArgumentCompletionView, DeleteConfirmationView, format_response_error @@ -280,7 +280,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: return filter_, filter_list, list_type = result - overrides_values, extra_fields_overrides = filter_overrides(filter_, filter_list, list_type) + overrides_values, extra_fields_overrides = filter_serializable_overrides(filter_) all_settings_repr_dict = build_filter_repr_dict( filter_list, list_type, type(filter_), overrides_values, extra_fields_overrides @@ -388,7 +388,7 @@ async def f_edit( return filter_, filter_list, list_type = result filter_type = type(filter_) - settings, filter_settings = filter_overrides(filter_, filter_list, list_type) + settings, filter_settings = filter_serializable_overrides(filter_) description, new_settings, new_filter_settings = description_and_settings_converter( filter_list, list_type, filter_type, From a7a04a118e1a9bf5a4d777ad43d40df9f035021c Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 17 Oct 2022 23:36:22 +0300 Subject: [PATCH 052/132] Add a command to query filters by settings --- .../filtering/_filter_lists/filter_list.py | 8 +- bot/exts/filtering/_filters/filter.py | 6 +- bot/exts/filtering/_settings.py | 7 + bot/exts/filtering/_ui/filter.py | 88 +++-- bot/exts/filtering/_ui/filter_list.py | 6 +- bot/exts/filtering/_ui/search.py | 365 ++++++++++++++++++ bot/exts/filtering/_ui/ui.py | 4 +- bot/exts/filtering/filtering.py | 127 +++++- bot/pagination.py | 7 +- 9 files changed, 563 insertions(+), 55 deletions(-) create mode 100644 bot/exts/filtering/_ui/search.py diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index f993665f27..84a43072b7 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -84,14 +84,14 @@ def filter_list_result(self, ctx: FilterContext) -> list[Filter]: return relevant_filters - def default(self, setting: str) -> Any: + def default(self, setting_name: str) -> Any: """Get the default value of a specific setting.""" missing = object() - value = self.defaults.actions.get_setting(setting, missing) + value = self.defaults.actions.get_setting(setting_name, missing) if value is missing: - value = self.defaults.validations.get_setting(setting, missing) + value = self.defaults.validations.get_setting(setting_name, missing) if value is missing: - raise ValueError(f"Couldn't find a setting named {setting!r}.") + raise ValueError(f"Couldn't find a setting named {setting_name!r}.") return value diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 0d11d5b3cb..0957997816 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, Optional +from typing import Any from pydantic import ValidationError @@ -28,7 +28,7 @@ def __init__(self, filter_data: dict, defaults: Defaults | None = None): self.description = filter_data["description"] self.actions, self.validations = create_settings(filter_data["settings"], defaults=defaults) if self.extra_fields_type: - self.extra_fields = self.extra_fields_type.parse_raw(filter_data["additional_field"]) + self.extra_fields = self.extra_fields_type.parse_raw(filter_data["additional_field"] or "{}") # noqa: P103 else: self.extra_fields = None @@ -52,7 +52,7 @@ def triggered_on(self, ctx: FilterContext) -> bool: """Search for the filter's content within a given context.""" @classmethod - def validate_filter_settings(cls, extra_fields: dict) -> tuple[bool, Optional[str]]: + def validate_filter_settings(cls, extra_fields: dict) -> tuple[bool, str | None]: """Validate whether the supplied fields are valid for the filter, and provide the error message if not.""" if cls.extra_fields_type is None: return True, None diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index 4c2114f07f..066c7a369d 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -211,3 +211,10 @@ class Defaults(NamedTuple): actions: ActionSettings validations: ValidationSettings + + def dict(self) -> dict[str, Any]: + """Return a dict representation of the stored fields across all entries.""" + dict_ = {} + for settings in self: + dict_ = reduce(operator.or_, (entry.dict() for entry in settings.values()), dict_) + return dict_ diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index 765fba683d..37584e9fd6 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -28,7 +28,7 @@ def build_filter_repr_dict( settings_overrides: dict, extra_fields_overrides: dict ) -> dict: - """Build a dictionary of field names and values to pass to `_build_embed_from_dict`.""" + """Build a dictionary of field names and values to pass to `populate_embed_from_dict`.""" # Get filter list settings default_setting_values = {} for settings_group in filter_list[list_type].defaults: @@ -155,16 +155,16 @@ def __init__( ) self.add_item(add_select) - override_names = ( - list(settings_overrides) + [f"{filter_list.name}/{setting}" for setting in filter_settings_overrides] - ) - remove_select = CustomCallbackSelect( - self._remove_override, - placeholder="Select an override to remove", - options=[SelectOption(label=name) for name in sorted(override_names)], - row=2 - ) - if remove_select.options: + if settings_overrides or filter_settings_overrides: + override_names = ( + list(settings_overrides) + [f"{filter_list.name}/{setting}" for setting in filter_settings_overrides] + ) + remove_select = CustomCallbackSelect( + self._remove_override, + placeholder="Select an override to remove", + options=[SelectOption(label=name) for name in sorted(override_names)], + row=2 + ) self.add_item(remove_select) @discord.ui.button(label="Edit Content", row=3) @@ -285,9 +285,9 @@ async def update_embed( dict_to_edit[setting_name] = setting_value # If there's already an override, remove it, since the new value is the same as the default. elif setting_name in dict_to_edit: - del dict_to_edit[setting_name] + dict_to_edit.pop(setting_name) elif setting_name in dict_to_edit: - del dict_to_edit[setting_name] + dict_to_edit.pop(setting_name) # This is inefficient, but otherwise the selects go insane if the user attempts to edit the same setting # multiple times, even when replacing the select with a new one. @@ -315,8 +315,10 @@ async def edit_setting_override(self, interaction: Interaction, setting_name: st async def apply_template(self, template_id: str, embed_message: discord.Message, interaction: Interaction) -> None: """Replace any non-overridden settings with overrides from the given filter.""" try: - settings, filter_settings = template_settings(template_id, self.filter_list, self.list_type) - except ValueError as e: # The interaction is necessary to send an ephemeral message. + settings, filter_settings = template_settings( + template_id, self.filter_list, self.list_type, self.filter_type + ) + except BadArgument as e: # The interaction object is necessary to send an ephemeral message. await interaction.response.send_message(f":x: {e}", ephemeral=True) return else: @@ -326,6 +328,7 @@ async def apply_template(self, template_id: str, embed_message: discord.Message, self.filter_settings_overrides = filter_settings | self.filter_settings_overrides self.embed.clear_fields() await embed_message.edit(embed=self.embed, view=self.copy()) + self.stop() async def _remove_override(self, interaction: Interaction, select: discord.ui.Select) -> None: """ @@ -380,28 +383,7 @@ def description_and_settings_converter( filter_settings = {} for setting, _ in list(settings.items()): - if setting not in loaded_settings: - # It's a filter setting - if "/" in setting: - setting_list_name, filter_setting_name = setting.split("/", maxsplit=1) - if setting_list_name.lower() != filter_list.name.lower(): - raise BadArgument( - f"A setting for a {setting_list_name!r} filter was provided, " - f"but the list name is {filter_list.name!r}" - ) - if filter_setting_name not in loaded_filter_settings[filter_list.name]: - raise BadArgument(f"{setting!r} is not a recognized setting.") - type_ = loaded_filter_settings[filter_list.name][filter_setting_name][2] - try: - parsed_value = parse_value(settings.pop(setting), type_) - if not repr_equals(parsed_value, getattr(filter_type.extra_fields_type(), filter_setting_name)): - filter_settings[filter_setting_name] = parsed_value - except (TypeError, ValueError) as e: - raise BadArgument(e) - else: - raise BadArgument(f"{setting!r} is not a recognized setting.") - # It's a filter list setting - else: + if setting in loaded_settings: # It's a filter list setting type_ = loaded_settings[setting][2] try: parsed_value = parse_value(settings.pop(setting), type_) @@ -409,11 +391,28 @@ def description_and_settings_converter( settings[setting] = parsed_value except (TypeError, ValueError) as e: raise BadArgument(e) + elif "/" not in setting: + raise BadArgument(f"{setting!r} is not a recognized setting.") + else: # It's a filter setting + filter_name, filter_setting_name = setting.split("/", maxsplit=1) + if filter_name.lower() != filter_type.name.lower(): + raise BadArgument( + f"A setting for a {filter_name!r} filter was provided, but the filter name is {filter_type.name!r}" + ) + if filter_setting_name not in loaded_filter_settings[filter_type.name]: + raise BadArgument(f"{setting!r} is not a recognized setting.") + type_ = loaded_filter_settings[filter_type.name][filter_setting_name][2] + try: + parsed_value = parse_value(settings.pop(setting), type_) + if not repr_equals(parsed_value, getattr(filter_type.extra_fields_type(), filter_setting_name)): + filter_settings[filter_setting_name] = parsed_value + except (TypeError, ValueError) as e: + raise BadArgument(e) # Pull templates settings and apply them. if template is not None: try: - t_settings, t_filter_settings = template_settings(template, filter_list, list_type) + t_settings, t_filter_settings = template_settings(template, filter_list, list_type, filter_type) except ValueError as e: raise BadArgument(str(e)) else: @@ -430,18 +429,25 @@ def filter_serializable_overrides(filter_: Filter) -> tuple[dict, dict]: return to_serializable(overrides_values), to_serializable(extra_fields_overrides) -def template_settings(filter_id: str, filter_list: FilterList, list_type: ListType) -> tuple[dict, dict]: +def template_settings( + filter_id: str, filter_list: FilterList, list_type: ListType, filter_type: type[Filter] +) -> tuple[dict, dict]: """Find the filter with specified ID, and return its settings.""" try: filter_id = int(filter_id) if filter_id < 0: raise ValueError() except ValueError: - raise ValueError("Template value must be a non-negative integer.") + raise BadArgument("Template value must be a non-negative integer.") if filter_id not in filter_list[list_type].filters: - raise ValueError( + raise BadArgument( f"Could not find filter with ID `{filter_id}` in the {list_type.name} {filter_list.name} list." ) filter_ = filter_list[list_type].filters[filter_id] + + if not isinstance(filter_, filter_type): + raise BadArgument( + f"The template filter name is {filter_.name!r}, but the target filter is {filter_type.name!r}" + ) return filter_serializable_overrides(filter_) diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index 15d81322b2..e77e29ec91 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -24,7 +24,11 @@ def settings_converter(loaded_settings: dict, input_data: str) -> dict[str, Any] if not parsed: return {} - settings = {setting: value for setting, value in [part.split("=", maxsplit=1) for part in parsed]} + try: + settings = {setting: value for setting, value in [part.split("=", maxsplit=1) for part in parsed]} + except ValueError: + raise BadArgument("The settings provided are not in the correct format.") + for setting in settings: if setting not in loaded_settings: raise BadArgument(f"{setting!r} is not a recognized setting.") diff --git a/bot/exts/filtering/_ui/search.py b/bot/exts/filtering/_ui/search.py new file mode 100644 index 0000000000..d553c28eaa --- /dev/null +++ b/bot/exts/filtering/_ui/search.py @@ -0,0 +1,365 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import discord +from discord import Interaction, SelectOption +from discord.ext.commands import BadArgument + +from bot.exts.filtering._filter_lists import FilterList, ListType +from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._settings_types.settings_entry import SettingsEntry +from bot.exts.filtering._ui.filter import filter_serializable_overrides +from bot.exts.filtering._ui.ui import ( + COMPONENT_TIMEOUT, CustomCallbackSelect, EditBaseView, MISSING, SETTINGS_DELIMITER, parse_value, + populate_embed_from_dict +) + + +def search_criteria_converter( + filter_lists: dict, + loaded_filters: dict, + loaded_settings: dict, + loaded_filter_settings: dict, + filter_type: type[Filter] | None, + input_data: str +) -> tuple[dict[str, Any], dict[str, Any], type[Filter]]: + """Parse a string representing setting overrides, and validate the setting names.""" + if not input_data: + return {}, {}, filter_type + + parsed = SETTINGS_DELIMITER.split(input_data) + if not parsed: + return {}, {}, filter_type + + try: + settings = {setting: value for setting, value in [part.split("=", maxsplit=1) for part in parsed]} + except ValueError: + raise BadArgument("The settings provided are not in the correct format.") + + template = None + if "--template" in settings: + template = settings.pop("--template") + + filter_settings = {} + for setting, _ in list(settings.items()): + if setting in loaded_settings: # It's a filter list setting + type_ = loaded_settings[setting][2] + try: + settings[setting] = parse_value(settings[setting], type_) + except (TypeError, ValueError) as e: + raise BadArgument(e) + elif "/" not in setting: + raise BadArgument(f"{setting!r} is not a recognized setting.") + else: # It's a filter setting + filter_name, filter_setting_name = setting.split("/", maxsplit=1) + if not filter_type: + if filter_name in loaded_filters: + filter_type = loaded_filters[filter_name] + else: + raise BadArgument(f"There's no filter type named {filter_name!r}.") + if filter_name.lower() != filter_type.name.lower(): + raise BadArgument( + f"A setting for a {filter_name!r} filter was provided, " + f"but the filter name is {filter_type.name!r}" + ) + if filter_setting_name not in loaded_filter_settings[filter_type.name]: + raise BadArgument(f"{setting!r} is not a recognized setting.") + type_ = loaded_filter_settings[filter_type.name][filter_setting_name][2] + try: + filter_settings[filter_setting_name] = parse_value(settings.pop(setting), type_) + except (TypeError, ValueError) as e: + raise BadArgument(e) + + # Pull templates settings and apply them. + if template is not None: + try: + t_settings, t_filter_settings, filter_type = template_settings(template, filter_lists, filter_type) + except ValueError as e: + raise BadArgument(str(e)) + else: + # The specified settings go on top of the template + settings = t_settings | settings + filter_settings = t_filter_settings | filter_settings + + return settings, filter_settings, filter_type + + +def get_filter(filter_id: int, filter_lists: dict) -> tuple[Filter, FilterList, ListType] | None: + """Return a filter with the specific filter_id, if found.""" + for filter_list in filter_lists.values(): + for list_type, sublist in filter_list.items(): + if filter_id in sublist.filters: + return sublist.filters[filter_id], filter_list, list_type + return None + + +def template_settings( + filter_id: str, filter_lists: dict, filter_type: type[Filter] | None +) -> tuple[dict, dict, type[Filter]]: + """Find a filter with the specified ID and filter type, and return its settings and (maybe newly found) type.""" + try: + filter_id = int(filter_id) + if filter_id < 0: + raise ValueError() + except ValueError: + raise BadArgument("Template value must be a non-negative integer.") + + result = get_filter(filter_id, filter_lists) + if not result: + raise BadArgument(f"Could not find a filter with ID `{filter_id}`.") + filter_, filter_list, list_type = result + + if filter_type and not isinstance(filter_, filter_type): + raise BadArgument(f"The filter with ID `{filter_id}` is not of type {filter_type.name!r}.") + + settings, filter_settings = filter_serializable_overrides(filter_) + return settings, filter_settings, type(filter_) + + +def build_search_repr_dict( + settings: dict[str, Any], filter_settings: dict[str, Any], filter_type: type[Filter] | None +) -> dict: + """Build a dictionary of field names and values to pass to `populate_embed_from_dict`.""" + total_values = settings.copy() + if filter_type: + for setting_name, value in filter_settings.items(): + total_values[f"{filter_type.name}/{setting_name}"] = value + + return total_values + + +class SearchEditView(EditBaseView): + """A view used to edit the search criteria before performing the search.""" + + class _REMOVE: + """Sentinel value for when an override should be removed.""" + + def __init__( + self, + filter_type: type[Filter] | None, + settings: dict[str, Any], + filter_settings: dict[str, Any], + loaded_filter_lists: dict[str, FilterList], + loaded_filters: dict[str, type[Filter]], + loaded_settings: dict[str, tuple[str, SettingsEntry, type]], + loaded_filter_settings: dict[str, dict[str, tuple[str, SettingsEntry, type]]], + author: discord.User | discord.Member, + embed: discord.Embed, + confirm_callback: Callable + ): + super().__init__(author) + self.filter_type = filter_type + self.settings = settings + self.filter_settings = filter_settings + self.loaded_filter_lists = loaded_filter_lists + self.loaded_filters = loaded_filters + self.loaded_settings = loaded_settings + self.loaded_filter_settings = loaded_filter_settings + self.embed = embed + self.confirm_callback = confirm_callback + + title = "Filters Search" + if filter_type: + title += f" - {filter_type.name.title()}" + embed.set_author(name=title) + + settings_repr_dict = build_search_repr_dict(settings, filter_settings, filter_type) + populate_embed_from_dict(embed, settings_repr_dict) + + self.type_per_setting_name = {setting: info[2] for setting, info in loaded_settings.items()} + if filter_type: + self.type_per_setting_name.update({ + f"{filter_type.name}/{name}": type_ + for name, (_, _, type_) in loaded_filter_settings.get(filter_type.name, {}).items() + }) + + add_select = CustomCallbackSelect( + self._prompt_new_value, + placeholder="Add or edit criterion", + options=[SelectOption(label=name) for name in sorted(self.type_per_setting_name)], + row=0 + ) + self.add_item(add_select) + + if settings_repr_dict: + remove_select = CustomCallbackSelect( + self._remove_criterion, + placeholder="Select a criterion to remove", + options=[SelectOption(label=name) for name in sorted(settings_repr_dict)], + row=1 + ) + self.add_item(remove_select) + + @discord.ui.button(label="Template", row=2) + async def enter_template(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to enter a filter template ID and copy its overrides over.""" + modal = TemplateModal(self, interaction.message) + await interaction.response.send_modal(modal) + + @discord.ui.button(label="Filter Type", row=2) + async def enter_filter_type(self, interaction: Interaction, button: discord.ui.Button) -> None: + """A button to enter a filter type.""" + modal = FilterTypeModal(self, interaction.message) + await interaction.response.send_modal(modal) + + @discord.ui.button(label="✅ Confirm", style=discord.ButtonStyle.green, row=3) + async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Confirm the search criteria and perform the search.""" + await interaction.response.edit_message(view=None) # Make sure the interaction succeeds first. + try: + await self.confirm_callback(interaction.message, self.filter_type, self.settings, self.filter_settings) + except BadArgument as e: + await interaction.message.reply( + embed=discord.Embed(colour=discord.Colour.red(), title="Bad Argument", description=str(e)) + ) + await interaction.message.edit(view=self) + else: + self.stop() + + @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red, row=3) + async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Cancel the operation.""" + await interaction.response.edit_message(content="🚫 Operation canceled.", embed=None, view=None) + self.stop() + + def current_value(self, setting_name: str) -> Any: + """Get the current value stored for the setting or MISSING if none found.""" + if setting_name in self.settings: + return self.settings[setting_name] + if "/" in setting_name: + _, setting_name = setting_name.split("/", maxsplit=1) + if setting_name in self.filter_settings: + return self.filter_settings[setting_name] + return MISSING + + async def update_embed( + self, + interaction_or_msg: discord.Interaction | discord.Message, + *, + setting_name: str | None = None, + setting_value: str | type[SearchEditView._REMOVE] | None = None, + ) -> None: + """ + Update the embed with the new information. + + If a setting name is provided with a _REMOVE value, remove the override. + If `interaction_or_msg` is a Message, the invoking Interaction must be deferred before calling this function. + """ + if not setting_name: # Can be None just to make the function signature compatible with the parent class. + return + + if "/" in setting_name: + filter_name, setting_name = setting_name.split("/", maxsplit=1) + dict_to_edit = self.filter_settings + else: + dict_to_edit = self.settings + + # Update the criterion value or remove it + if setting_value is not self._REMOVE: + dict_to_edit[setting_name] = setting_value + elif setting_name in dict_to_edit: + dict_to_edit.pop(setting_name) + + self.embed.clear_fields() + new_view = self.copy() + + try: + if isinstance(interaction_or_msg, discord.Interaction): + await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) + else: + await interaction_or_msg.edit(embed=self.embed, view=new_view) + except discord.errors.HTTPException: # Just in case of faulty input. + pass + else: + self.stop() + + async def _remove_criterion(self, interaction: Interaction, select: discord.ui.Select) -> None: + """ + Remove the criterion the user selected, and edit the embed. + + The interaction needs to be the selection of the setting attached to the embed. + """ + await self.update_embed(interaction, setting_name=select.values[0], setting_value=self._REMOVE) + + async def apply_template(self, template_id: str, embed_message: discord.Message, interaction: Interaction) -> None: + """Set any unset criteria with settings values from the given filter.""" + try: + settings, filter_settings, self.filter_type = template_settings( + template_id, self.loaded_filter_lists, self.filter_type + ) + except BadArgument as e: # The interaction object is necessary to send an ephemeral message. + await interaction.response.send_message(f":x: {e}", ephemeral=True) + return + else: + await interaction.response.defer() + + self.settings = settings | self.settings + self.filter_settings = filter_settings | self.filter_settings + self.embed.clear_fields() + await embed_message.edit(embed=self.embed, view=self.copy()) + self.stop() + + async def apply_filter_type(self, type_name: str, embed_message: discord.Message, interaction: Interaction) -> None: + """Set a new filter type and reset any criteria for settings of the old filter type.""" + if type_name.lower() not in self.loaded_filters: + if type_name.lower()[:-1] not in self.loaded_filters: # In case the user entered the plural form. + await interaction.response.send_message(f":x: No such filter type {type_name!r}.", ephemeral=True) + return + type_name = type_name[:-1] + type_name = type_name.lower() + await interaction.response.defer() + + if self.filter_type and type_name == self.filter_type.name: + return + self.filter_type = self.loaded_filters[type_name] + self.filter_settings = {} + self.embed.clear_fields() + await embed_message.edit(embed=self.embed, view=self.copy()) + self.stop() + + def copy(self) -> SearchEditView: + """Create a copy of this view.""" + return SearchEditView( + self.filter_type, + self.settings, + self.filter_settings, + self.loaded_filter_lists, + self.loaded_filters, + self.loaded_settings, + self.loaded_filter_settings, + self.author, + self.embed, + self.confirm_callback + ) + + +class TemplateModal(discord.ui.Modal, title="Template"): + """A modal to enter a filter ID to copy its overrides over.""" + + template = discord.ui.TextInput(label="Template Filter ID", required=False) + + def __init__(self, embed_view: SearchEditView, message: discord.Message): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.embed_view = embed_view + self.message = message + + async def on_submit(self, interaction: Interaction) -> None: + """Update the embed with the new description.""" + await self.embed_view.apply_template(self.template.value, self.message, interaction) + + +class FilterTypeModal(discord.ui.Modal, title="Template"): + """A modal to enter a filter ID to copy its overrides over.""" + + filter_type = discord.ui.TextInput(label="Filter Type") + + def __init__(self, embed_view: SearchEditView, message: discord.Message): + super().__init__(timeout=COMPONENT_TIMEOUT) + self.embed_view = embed_view + self.message = message + + async def on_submit(self, interaction: Interaction) -> None: + """Update the embed with the new description.""" + await self.embed_view.apply_filter_type(self.filter_type.value, self.message, interaction) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 980eba02af..c506db1fe5 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -34,7 +34,7 @@ MAX_EMBED_DESCRIPTION = 4000 SETTINGS_DELIMITER = re.compile(r"\s+(?=\S+=\S+)") -SINGLE_SETTING_PATTERN = re.compile(r"\w+=.+") +SINGLE_SETTING_PATTERN = re.compile(r"[\w/]+=.+") # Sentinel value to denote that a value is missing MISSING = object() @@ -76,7 +76,7 @@ def parse_value(value: str, type_: type[T]) -> T: if type_ in (tuple, list, set): return type_(value.split(",")) if type_ is bool: - return value == "True" + return value.lower() == "true" or value == "1" if isinstance(type_, EnumMeta): return type_[value.upper()] diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 6ff5181a92..890b25718e 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -26,8 +26,9 @@ build_filter_repr_dict, description_and_settings_converter, filter_serializable_overrides, populate_embed_from_dict ) from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter +from bot.exts.filtering._ui.search import SearchEditView, search_criteria_converter from bot.exts.filtering._ui.ui import ArgumentCompletionView, DeleteConfirmationView, format_response_error -from bot.exts.filtering._utils import past_tense, starting_value, to_serializable +from bot.exts.filtering._utils import past_tense, repr_equals, starting_value, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.messages import format_channel, format_user @@ -523,6 +524,63 @@ async def f_match(self, ctx: Context, message: Message | None, *, string: str | embed = Embed(colour=Colour.blue(), title="Match results") await LinePaginator.paginate(lines, ctx, embed, max_lines=10, empty=False) + @filter.command(name="search") + async def f_search( + self, + ctx: Context, + noui: Literal["noui"] | None, + filter_type_name: str | None, + *, + settings: str = "" + ) -> None: + """ + Find filters with the provided settings. The format is identical to that of the add and edit commands. + + If a list type and/or a list name are provided, the search will be limited to those parameters. A list name must + be provided in order to search by filter-specific settings. + """ + filter_type = None + if filter_type_name: + filter_type_name = filter_type_name.lower() + filter_type = self.loaded_filters.get(filter_type_name) + if not filter_type: + self.loaded_filters.get(filter_type_name[:-1]) # In case the user tried to specify the plural form. + # If settings were provided with no filter_type, discord.py will capture the first word as the filter type. + if filter_type is None and filter_type_name is not None: + if settings: + settings = f"{filter_type_name} {settings}" + else: + settings = filter_type_name + filter_type_name = None + + settings, filter_settings, filter_type = search_criteria_converter( + self.filter_lists, + self.loaded_filters, + self.loaded_settings, + self.loaded_filter_settings, + filter_type, + settings + ) + + if noui: + await self._search_filters(ctx.message, filter_type, settings, filter_settings) + return + + embed = Embed(colour=Colour.blue()) + view = SearchEditView( + filter_type, + settings, + filter_settings, + self.filter_lists, + self.loaded_filters, + self.loaded_settings, + self.loaded_filter_settings, + ctx.author, + embed, + self._search_filters + ) + await ctx.send(embed=embed, reference=ctx.message, view=view) + # endregion # region: filterlist group @@ -787,7 +845,7 @@ async def _send_list(ctx: Context, filter_list: FilterList, list_type: ListType) embed = Embed(colour=Colour.blue()) embed.set_author(name=f"List of {filter_list[list_type].label}s ({len(lines)} total)") - await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) + await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False, reply=True) def _get_filter_by_id(self, id_: int) -> Optional[tuple[Filter, FilterList, ListType]]: """Get the filter object corresponding to the provided ID, along with its containing list and list type.""" @@ -954,6 +1012,71 @@ async def _patch_filter_list(msg: Message, filter_list: FilterList, list_type: L filter_list.add_list(response) await msg.reply(f"✅ Edited filter list: {filter_list[list_type].label}") + def _filter_match_query( + self, filter_: Filter, settings_query: dict, filter_settings_query: dict, differ_by_default: set[str] + ) -> bool: + """Return whether the given filter matches the query.""" + override_matches = set() + overrides, _ = filter_.overrides + for setting_name, setting_value in settings_query.items(): + if setting_name not in overrides: + continue + if repr_equals(overrides[setting_name], setting_value): + override_matches.add(setting_name) + else: # If an override doesn't match then the filter doesn't match. + return False + if not (differ_by_default <= override_matches): # The overrides didn't cover for the default mismatches. + return False + + filter_settings = filter_.extra_fields.dict() if filter_.extra_fields else {} + # If the dict changes then some fields were not the same. + return (filter_settings | filter_settings_query) == filter_settings + + def _search_filter_list( + self, atomic_list: AtomicList, filter_type: type[Filter] | None, settings: dict, filter_settings: dict + ) -> list[Filter]: + """Find all filters in the filter list which match the settings.""" + # If the default answers are known, only the overrides need to be checked for each filter. + all_defaults = atomic_list.defaults.dict() + match_by_default = set() + differ_by_default = set() + for setting_name, setting_value in settings.items(): + if repr_equals(all_defaults[setting_name], setting_value): + match_by_default.add(setting_name) + else: + differ_by_default.add(setting_name) + + result_filters = [] + for filter_ in atomic_list.filters.values(): + if filter_type and not isinstance(filter_, filter_type): + continue + if self._filter_match_query(filter_, settings, filter_settings, differ_by_default): + result_filters.append(filter_) + + return result_filters + + async def _search_filters( + self, message: Message, filter_type: type[Filter] | None, settings: dict, filter_settings: dict + ) -> None: + """Find all filters which match the settings and display them.""" + lines = [] + result_count = 0 + for filter_list in self.filter_lists.values(): + if filter_type and filter_type not in filter_list.filter_types: + continue + for atomic_list in filter_list.values(): + list_results = self._search_filter_list(atomic_list, filter_type, settings, filter_settings) + if list_results: + lines.append(f"**{atomic_list.label.title()}**") + lines.extend(map(str, list_results)) + lines.append("") + result_count += len(list_results) + + embed = Embed(colour=Colour.blue()) + embed.set_author(name=f"Search Results ({result_count} total)") + ctx = await bot.instance.get_context(message) + await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False, reply=True) + # endregion diff --git a/bot/pagination.py b/bot/pagination.py index 10bef1c9f9..92fa781eeb 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -204,6 +204,7 @@ async def paginate( footer_text: str = None, url: str = None, exception_on_empty_embed: bool = False, + reply: bool = False, ) -> t.Optional[discord.Message]: """ Use a paginator and set of reactions to provide pagination over a set of lines. @@ -251,6 +252,8 @@ async def paginate( embed.description = paginator.pages[current_page] + reference = ctx.message if reply else None + if len(paginator.pages) <= 1: if footer_text: embed.set_footer(text=footer_text) @@ -261,7 +264,7 @@ async def paginate( log.trace(f"Setting embed url to '{url}'") log.debug("There's less than two pages, so we won't paginate - sending single page on its own") - return await ctx.send(embed=embed) + return await ctx.send(embed=embed, reference=reference) else: if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") @@ -274,7 +277,7 @@ async def paginate( log.trace(f"Setting embed url to '{url}'") log.debug("Sending first page to channel...") - message = await ctx.send(embed=embed) + message = await ctx.send(embed=embed, reference=reference) log.debug("Adding emoji reactions to message...") From 5a4aa1f51875ce12e4484a31b5bdabb2ef659059 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 22 Oct 2022 21:52:50 +0300 Subject: [PATCH 053/132] Add everyone filter, fix invite filtering - Add the everyone filter, from a new class of filters - UniqueFilter. These filters should only be run once per context. These filters can modify any part of their behavior, including which events they respond to. - The invite filtering had a bug where it could exit early when the context wasn't a valid whitelisting context (which usually means that any unkown filters are allowed), despite there possibly being blocked filters which require addressing. --- bot/exts/filtering/_filter_lists/domain.py | 20 +--- bot/exts/filtering/_filter_lists/extension.py | 2 +- .../filtering/_filter_lists/filter_list.py | 104 +++++++++++++--- bot/exts/filtering/_filter_lists/invite.py | 113 +++++++++--------- bot/exts/filtering/_filter_lists/token.py | 20 +--- bot/exts/filtering/_filter_lists/unique.py | 75 ++++++++++++ bot/exts/filtering/_filters/filter.py | 14 ++- .../filtering/_filters/unique/__init__.py | 9 ++ .../filtering/_filters/unique/everyone.py | 28 +++++ bot/exts/filtering/_utils.py | 2 +- bot/exts/filtering/filtering.py | 8 +- 11 files changed, 285 insertions(+), 110 deletions(-) create mode 100644 bot/exts/filtering/_filter_lists/unique.py create mode 100644 bot/exts/filtering/_filters/unique/__init__.py create mode 100644 bot/exts/filtering/_filters/unique/everyone.py diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index 17984e276d..d97aa252ba 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -2,8 +2,6 @@ import re import typing -from functools import reduce -from operator import or_ from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType @@ -18,7 +16,7 @@ URL_RE = re.compile(r"https?://(\S+)", flags=re.IGNORECASE) -class DomainsList(FilterList): +class DomainsList(FilterList[DomainFilter]): """ A list of filters, each looking for a specific domain given by URL. @@ -59,18 +57,6 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, actions = None messages = [] if triggers: - action_defaults = self[ListType.DENY].defaults.actions - actions = reduce( - or_, - (filter_.actions.fallback_to(action_defaults) if filter_.actions else action_defaults - for filter_ in triggers - ) - ) - if len(triggers) == 1: - message = f"#{triggers[0].id} (`{triggers[0].content}`)" - if triggers[0].description: - message += f" - {triggers[0].description}" - messages = [message] - else: - messages = [f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers] + actions = self[ListType.DENY].merge_actions(triggers) + messages = self[ListType.DENY].format_messages(triggers) return actions, messages diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index a58c6c45e6..3f9d2b2870 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -33,7 +33,7 @@ ) -class ExtensionsList(FilterList): +class ExtensionsList(FilterList[ExtensionFilter]): """ A list of filters, each looking for a file attachment with a specific extension. diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 84a43072b7..55204335b7 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,11 +1,18 @@ +import dataclasses +import typing from abc import abstractmethod +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass from enum import Enum -from typing import Any, NamedTuple +from functools import reduce +from operator import or_ +from typing import Any from discord.ext.commands import BadArgument -from bot.exts.filtering._filter_context import FilterContext -from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import Filter, UniqueFilter from bot.exts.filtering._settings import ActionSettings, Defaults, create_settings from bot.exts.filtering._utils import FieldRequiring, past_tense from bot.log import get_logger @@ -36,7 +43,8 @@ def list_type_converter(argument: str) -> ListType: raise BadArgument(f"No matching list type found for {argument!r}.") -class AtomicList(NamedTuple): +@dataclass(frozen=True) +class AtomicList: """ Represents the atomic structure of a single filter list as it appears in the database. @@ -68,11 +76,16 @@ def filter_list_result(self, ctx: FilterContext) -> list[Filter]: If the filter is relevant in context, see if it actually triggers. """ - passed_by_default, failed_by_default = self.defaults.validations.evaluate(ctx) + return self._create_filter_list_result(ctx, self.defaults, self.filters.values()) + + @staticmethod + def _create_filter_list_result(ctx: FilterContext, defaults: Defaults, filters: Iterable[Filter]) -> list[Filter]: + """A helper function to evaluate the result of `filter_list_result`.""" + passed_by_default, failed_by_default = defaults.validations.evaluate(ctx) default_answer = not bool(failed_by_default) relevant_filters = [] - for filter_ in self.filters.values(): + for filter_ in filters: if not filter_.validations: if default_answer and filter_.triggered_on(ctx): relevant_filters.append(filter_) @@ -94,8 +107,36 @@ def default(self, setting_name: str) -> Any: raise ValueError(f"Couldn't find a setting named {setting_name!r}.") return value + def merge_actions(self, filters: list[Filter]) -> ActionSettings | None: + """ + Merge the settings of the given filters, with the list's defaults as fallback. + + If `merge_default` is True, include it in the merge instead of using it as a fallback. + """ + try: + result = reduce(or_, (filter_.actions for filter_ in filters if filter_.actions)) + except TypeError: # The sequence fed to reduce is empty. + return None + + return result.fallback_to(self.defaults.actions) + + @staticmethod + def format_messages(triggers: list[Filter], *, expand_single_filter: bool = True) -> list[str]: + """Convert the filters into strings that can be added to the alert embed.""" + if len(triggers) == 1 and expand_single_filter: + message = f"#{triggers[0].id} (`{triggers[0].content}`)" + if triggers[0].description: + message += f" - {triggers[0].description}" + messages = [message] + else: + messages = [f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers] + return messages + + +T = typing.TypeVar("T", bound=Filter) + -class FilterList(FieldRequiring, dict[ListType, AtomicList]): +class FilterList(dict[ListType, AtomicList], typing.Generic[T], FieldRequiring): """Dispatches events to lists of _filters, and aggregates the responses into a single list of actions to take.""" # Each subclass must define a name matching the filter_list name we're expecting to receive from the database. @@ -110,39 +151,70 @@ def add_list(self, list_data: dict) -> AtomicList: filters = {} for filter_data in list_data["filters"]: - filters[filter_data["id"]] = self._create_filter(filter_data, defaults) + new_filter = self._create_filter(filter_data, defaults) + if new_filter: + filters[filter_data["id"]] = new_filter self[list_type] = AtomicList(list_data["id"], self.name, list_type, defaults, filters) return self[list_type] - def add_filter(self, list_type: ListType, filter_data: dict) -> Filter: + def add_filter(self, list_type: ListType, filter_data: dict) -> T | None: """Add a filter to the list of the specified type.""" new_filter = self._create_filter(filter_data, self[list_type].defaults) - self[list_type].filters[filter_data["id"]] = new_filter + if new_filter: + self[list_type].filters[filter_data["id"]] = new_filter return new_filter @abstractmethod - def get_filter_type(self, content: str) -> type[Filter]: + def get_filter_type(self, content: str) -> type[T]: """Get a subclass of filter matching the filter list and the filter's content.""" @property @abstractmethod - def filter_types(self) -> set[type[Filter]]: + def filter_types(self) -> set[type[T]]: """Return the types of filters used by this list.""" @abstractmethod async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" - def _create_filter(self, filter_data: dict, defaults: Defaults) -> Filter: + def _create_filter(self, filter_data: dict, defaults: Defaults) -> T | None: """Create a filter from the given data.""" try: filter_type = self.get_filter_type(filter_data["content"]) - new_filter = filter_type(filter_data, defaults) + if filter_type: + return filter_type(filter_data, defaults) + else: + return None except TypeError as e: log.warning(e) - else: - return new_filter def __hash__(self): return hash(id(self)) + + +@dataclass(frozen=True) +class SubscribingAtomicList(AtomicList): + """ + A base class for a list of unique filters. + + Unique filters are ones that should only be run once in a given context. + Each unique filter is subscribed to a subset of events to respond to. + """ + + subscriptions: defaultdict[Event, list[Filter]] = dataclasses.field(default_factory=lambda: defaultdict(list)) + + def subscribe(self, filter_: UniqueFilter, *events: Event) -> None: + """ + Subscribe a unique filter to the given events. + + The filter is added to a list for each event. When the event is triggered, the filter context will be + dispatched to the subscribed filters. + """ + for event in events: + if filter_ not in self.subscriptions[event]: + self.subscriptions[event].append(filter_) + + def filter_list_result(self, ctx: FilterContext) -> list[Filter]: + """Sift through the list of filters, and return only the ones which apply to the given context.""" + return self._create_filter_list_result(ctx, self.defaults, self.subscriptions[ctx.event]) diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index d35fdd4a43..0b84aec0e7 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -1,8 +1,6 @@ from __future__ import annotations import typing -from functools import reduce -from operator import or_ from botcore.utils.regex import DISCORD_INVITE from discord import Embed, Invite @@ -20,7 +18,7 @@ from bot.exts.filtering.filtering import Filtering -class InviteList(FilterList): +class InviteList(FilterList[InviteFilter]): """ A list of filters, each looking for guild invites to a specific guild. @@ -52,10 +50,6 @@ def filter_types(self) -> set[type[Filter]]: async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" - _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) - if failed: # There's no invite filtering in this context. - return None, [] - text = clean_input(ctx.content) # Avoid escape characters @@ -66,62 +60,73 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, if not invite_codes: return None, [] - # Sort the invites into three categories: - denied_by_default = dict() # Denied unless whitelisted. - allowed_by_default = dict() # Allowed unless blacklisted (partnered or verified servers). - disallowed_invites = dict() # Always denied (invalid invites). + _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) + # If the allowed list doesn't operate in the context, unknown invites are allowed. + check_if_allowed = not failed + + # Sort the invites into two categories: + invites_for_inspection = dict() # Found guild invites requiring further inspection. + unknown_invites = dict() # Either don't resolve or group DMs. for invite_code in invite_codes: try: invite = await bot.instance.fetch_invite(invite_code) except NotFound: - disallowed_invites[invite_code] = None + if check_if_allowed: + unknown_invites[invite_code] = None else: - if not invite.guild: - disallowed_invites[invite_code] = invite - else: - if "PARTNERED" in invite.guild.features or "VERIFIED" in invite.guild.features: - allowed_by_default[invite_code] = invite - else: - denied_by_default[invite_code] = invite - - # Add the disallowed by default unless they're whitelisted. - guilds_for_inspection = {invite.guild.id for invite in denied_by_default.values()} - new_ctx = ctx.replace(content=guilds_for_inspection) - allowed = { - filter_.content for filter_ in self[ListType.ALLOW].filters.values() if filter_.triggered_on(new_ctx) + if invite.guild: + invites_for_inspection[invite_code] = invite + elif check_if_allowed: # Group DM + unknown_invites[invite_code] = invite + + # Find any blocked invites + new_ctx = ctx.replace(content={invite.guild.id for invite in invites_for_inspection.values()}) + triggered = self[ListType.DENY].filter_list_result(new_ctx) + blocked_guilds = {filter_.content for filter_ in triggered} + blocked_invites = { + code: invite for code, invite in invites_for_inspection.items() if invite.guild.id in blocked_guilds } - disallowed_invites.update({ - invite_code: invite for invite_code, invite in denied_by_default.items() if invite.guild.id not in allowed - }) - - # Add the allowed by default only if they're blacklisted. - guilds_for_inspection = {invite.guild.id for invite in allowed_by_default.values()} - new_ctx = ctx.replace(content=guilds_for_inspection) - triggered = self[ListType.ALLOW].filter_list_result(new_ctx) - disallowed_invites.update({ - invite_code: invite for invite_code, invite in allowed_by_default.items() - if invite.guild.id in {filter_.content for filter_ in triggered} - }) - - if not disallowed_invites: + + # Remove the ones which are already confirmed as blocked, or otherwise ones which are partnered or verified. + invites_for_inspection = { + code: invite for code, invite in invites_for_inspection.items() + if invite.guild.id not in blocked_guilds + and "PARTNERED" not in invite.guild.features and "VERIFIED" not in invite.guild.features + } + + # Remove any remaining invites which are allowed + guilds_for_inspection = {invite.guild.id for invite in invites_for_inspection.values()} + + if check_if_allowed: # Whether unknown invites need to be checked. + new_ctx = ctx.replace(content=guilds_for_inspection) + allowed = { + filter_.content for filter_ in self[ListType.ALLOW].filters.values() if filter_.triggered_on(new_ctx) + } + unknown_invites.update({ + code: invite for code, invite in invites_for_inspection.items() if invite.guild.id not in allowed + }) + + if not triggered and not unknown_invites: return None, [] actions = None - if len(disallowed_invites) > len(triggered): # There are invites which weren't allowed but aren't blacklisted. - deny_defaults = self[ListType.DENY].defaults.actions - actions = reduce( - or_, - ( - filter_.actions.fallback_to(deny_defaults) if filter_.actions else deny_defaults - for filter_ in triggered - ), - self[ListType.ALLOW].defaults.actions - ) - elif triggered: - actions = reduce(or_, (filter_.actions for filter_ in triggered)) - ctx.matches += {match[0] for match in matches if match.group("invite") in disallowed_invites} - ctx.alert_embeds += (self._guild_embed(invite) for invite in disallowed_invites.values() if invite) - return actions, [f"`{invite}`" for invite in disallowed_invites] + if unknown_invites: # There are invites which weren't allowed but aren't explicitly blocked. + actions = self[ListType.ALLOW].defaults.actions + # Blocked invites come second so that their actions have preference. + if triggered: + if actions: + actions |= self[ListType.DENY].merge_actions(triggered) + else: + actions = self[ListType.DENY].merge_actions(triggered) + + blocked_invites |= unknown_invites + ctx.matches += {match[0] for match in matches if match.group("invite") in blocked_invites} + ctx.alert_embeds += (self._guild_embed(invite) for invite in blocked_invites.values() if invite) + messages = self[ListType.DENY].format_messages(triggered) + messages += [ + f"`{code} - {invite.guild.id}`" if invite else f"`{code}`" for code, invite in unknown_invites.items() + ] + return actions, messages @staticmethod def _guild_embed(invite: Invite) -> Embed: diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index 4b161d9b7d..c7d7cb4447 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -2,8 +2,6 @@ import re import typing -from functools import reduce -from operator import or_ from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType @@ -18,7 +16,7 @@ SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL) -class TokensList(FilterList): +class TokensList(FilterList[TokenFilter]): """ A list of filters, each looking for a specific token in the given content given as regex. @@ -59,20 +57,8 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, actions = None messages = [] if triggers: - action_defaults = self[ListType.DENY].defaults.actions - actions = reduce( - or_, - (filter_.actions.fallback_to(action_defaults) if filter_.actions else action_defaults - for filter_ in triggers - ) - ) - if len(triggers) == 1: - message = f"#{triggers[0].id} (`{triggers[0].content}`)" - if triggers[0].description: - message += f" - {triggers[0].description}" - messages = [message] - else: - messages = [f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers] + actions = self[ListType.DENY].merge_actions(triggers) + messages = self[ListType.DENY].format_messages(triggers) return actions, messages @staticmethod diff --git a/bot/exts/filtering/_filter_lists/unique.py b/bot/exts/filtering/_filter_lists/unique.py new file mode 100644 index 0000000000..63caa7d36b --- /dev/null +++ b/bot/exts/filtering/_filter_lists/unique.py @@ -0,0 +1,75 @@ +from botcore.utils.logging import get_logger +from discord.ext.commands import Cog + +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType, SubscribingAtomicList +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._filters.unique import unique_filter_types +from bot.exts.filtering._settings import ActionSettings, Defaults, create_settings + +log = get_logger(__name__) + + +class UniquesList(FilterList[UniqueFilter]): + """ + A list of unique filters. + + Unique filters are ones that should only be run once in a given context. + Each unique filter subscribes to a subset of events to respond to. + """ + + name = "unique" + _already_warned = set() + + def __init__(self, filtering_cog: Cog): + super().__init__() + self.filtering_cog = filtering_cog # This is typed as a Cog to avoid a circular import. + self.loaded_types: dict[str, type[UniqueFilter]] = {} + + def add_list(self, list_data: dict) -> SubscribingAtomicList: + """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" + actions, validations = create_settings(list_data["settings"], keep_empty=True) + list_type = ListType(list_data["list_type"]) + defaults = Defaults(actions, validations) + new_list = SubscribingAtomicList(list_data["id"], self.name, list_type, defaults, {}) + self[list_type] = new_list + + filters = {} + events = set() + for filter_data in list_data["filters"]: + new_filter = self._create_filter(filter_data, defaults) + if new_filter: + new_list.subscribe(new_filter, *new_filter.events) + filters[filter_data["id"]] = new_filter + self.loaded_types[new_filter.name] = type(new_filter) + events.update(new_filter.events) + + new_list.filters.update(filters) + if hasattr(self.filtering_cog, "subscribe"): # Subscribe the filter list to any new events found. + self.filtering_cog.subscribe(self, *events) + return new_list + + def get_filter_type(self, content: str) -> type[UniqueFilter] | None: + """Get a subclass of filter matching the filter list and the filter's content.""" + try: + return unique_filter_types[content] + except KeyError: + if content not in self._already_warned: + log.warn(f"A unique filter named {content} was supplied, but no matching implementation found.") + self._already_warned.add(content) + return None + + @property + def filter_types(self) -> set[type[UniqueFilter]]: + """Return the types of filters used by this list.""" + return set(self.loaded_types.values()) + + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" + triggers = self[ListType.DENY].filter_list_result(ctx) + actions = None + messages = [] + if triggers: + actions = self[ListType.DENY].merge_actions(triggers) + messages = self[ListType.DENY].format_messages(triggers) + return actions, messages diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 0957997816..b0d19d3a84 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,9 +1,9 @@ -from abc import abstractmethod +from abc import ABC, abstractmethod from typing import Any from pydantic import ValidationError -from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._settings import Defaults, create_settings from bot.exts.filtering._utils import FieldRequiring @@ -79,3 +79,13 @@ def __str__(self) -> str: if self.description: string += f" - {self.description}" return string + + +class UniqueFilter(Filter, ABC): + """ + Unique filters are ones that should only be run once in a given context. + + This is as opposed to say running many domain filters on the same message. + """ + + events: tuple[Event, ...] = FieldRequiring.MUST_SET diff --git a/bot/exts/filtering/_filters/unique/__init__.py b/bot/exts/filtering/_filters/unique/__init__.py new file mode 100644 index 0000000000..ce78d69228 --- /dev/null +++ b/bot/exts/filtering/_filters/unique/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname + +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._utils import subclasses_in_package + +unique_filter_types = subclasses_in_package(dirname(__file__), f"{__name__}.", UniqueFilter) +unique_filter_types = {filter_.name: filter_ for filter_ in unique_filter_types} + +__all__ = [unique_filter_types] diff --git a/bot/exts/filtering/_filters/unique/everyone.py b/bot/exts/filtering/_filters/unique/everyone.py new file mode 100644 index 0000000000..06d3a19bb3 --- /dev/null +++ b/bot/exts/filtering/_filters/unique/everyone.py @@ -0,0 +1,28 @@ +import re + +from bot.constants import Guild +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + +EVERYONE_PING_RE = re.compile(rf"@everyone|<@&{Guild.id}>|@here") +CODE_BLOCK_RE = re.compile( + r"(?P``?)[^`]+?(?P=delim)(?!`+)" # Inline codeblock + r"|```(.+?)```", # Multiline codeblock + re.DOTALL | re.MULTILINE +) + + +class EveryoneFilter(UniqueFilter): + """Filter messages which contain `@everyone` and `@here` tags outside a codeblock.""" + + name = "everyone" + events = (Event.MESSAGE, Event.MESSAGE_EDIT) + + def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + # First pass to avoid running re.sub on every message + if not EVERYONE_PING_RE.search(ctx.content): + return False + + content_without_codeblocks = CODE_BLOCK_RE.sub("", ctx.content) + return bool(EVERYONE_PING_RE.search(content_without_codeblocks)) diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index 7149f7254c..a38fa22e41 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -16,7 +16,7 @@ T = TypeVar('T') -def subclasses_in_package(package: str, prefix: str, parent: type) -> set[type]: +def subclasses_in_package(package: str, prefix: str, parent: T) -> set[T]: """Return all the subclasses of class `parent`, found in the top-level of `package`, given by absolute path.""" subclasses = set() diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 890b25718e..837cd45c12 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -954,8 +954,11 @@ async def _post_new_filter( } response = await bot.instance.api_client.post('bot/filter/filters', json=to_serializable(payload)) new_filter = filter_list.add_filter(list_type, response) - extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, new_filter) - await msg.reply(f"✅ Added filter: {new_filter}" + extra_msg) + if new_filter: + extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, new_filter) + await msg.reply(f"✅ Added filter: {new_filter}" + extra_msg) + else: + await msg.reply(":x: Could not create the filter. Are you sure it's implemented?") @staticmethod async def _patch_filter( @@ -990,6 +993,7 @@ async def _patch_filter( response = await bot.instance.api_client.patch( f'bot/filter/filters/{filter_.id}', json=to_serializable(payload) ) + # Return type can be None, but if it's being edited then it's not supposed to be. edited_filter = filter_list.add_filter(list_type, response) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, edited_filter) await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) From e86fe0b9bde9504a9cbc468bd0200ffe1233a47c Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 23 Oct 2022 01:11:59 +0300 Subject: [PATCH 054/132] Add webhook filter This adds a way to add custom coroutines to the context, to execute when the filtering result is actioned. --- bot/exts/filtering/_filter_context.py | 2 + bot/exts/filtering/_filters/unique/webhook.py | 58 +++++++++++++++++++ bot/exts/filtering/_settings.py | 11 +++- .../actions/delete_messages.py | 8 ++- .../actions/infraction_and_notification.py | 2 +- 5 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 bot/exts/filtering/_filters/unique/webhook.py diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 5e2f5b45b2..da7ba0c77c 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Coroutine from dataclasses import dataclass, field, replace from enum import Enum, auto from typing import Optional, Union @@ -34,6 +35,7 @@ class FilterContext: action_descriptions: list = field(default_factory=list) # What actions were taken matches: list = field(default_factory=list) # What exactly was found notification_domain: str = field(default_factory=str) # A domain to send the user for context + additional_actions: list[Coroutine] = field(default_factory=list) # Additional actions to perform def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" diff --git a/bot/exts/filtering/_filters/unique/webhook.py b/bot/exts/filtering/_filters/unique/webhook.py new file mode 100644 index 0000000000..10f27a922d --- /dev/null +++ b/bot/exts/filtering/_filters/unique/webhook.py @@ -0,0 +1,58 @@ +import re + +from botcore.utils.logging import get_logger + +import bot +from bot import constants +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.moderation.modlog import ModLog + +log = get_logger(__name__) + + +WEBHOOK_URL_RE = re.compile( + r"((?:https?://)?(?:ptb\.|canary\.)?discord(?:app)?\.com/api/webhooks/\d+/)\S+/?", + re.IGNORECASE +) + + +class WebhookFilter(UniqueFilter): + """Scan messages to detect Discord webhooks links.""" + + name = "webhook" + events = (Event.MESSAGE, Event.MESSAGE_EDIT) + + @property + def mod_log(self) -> ModLog | None: + """Get current instance of `ModLog`.""" + return bot.instance.get_cog("ModLog") + + def triggered_on(self, ctx: FilterContext) -> bool: + """Search for a webhook in the given content. If found, attempt to delete it.""" + matches = set(WEBHOOK_URL_RE.finditer(ctx.content)) + if not matches: + return False + + # Don't log this. + if mod_log := self.mod_log: + mod_log.ignore(constants.Event.message_delete, ctx.message.id) + + for i, match in enumerate(matches, start=1): + extra = "" if len(matches) == 1 else f" ({i})" + # Queue the webhook for deletion. + ctx.additional_actions.append(self._delete_webhook(ctx, match[0], extra)) + # Don't show the full webhook in places such as the mod alert. + ctx.content = ctx.content.replace(match[0], match[1] + "xxx") + + return True + + @staticmethod + async def _delete_webhook(ctx: FilterContext, webhook_url: str, extra_message: str) -> None: + """Delete the given webhook and update the filter context.""" + async with bot.instance.http_session.delete(webhook_url) as resp: + # The Discord API Returns a 204 NO CONTENT response on success. + if resp.status == 204: + ctx.action_descriptions.append("webhook deleted" + extra_message) + else: + ctx.action_descriptions.append("failed to delete webhook" + extra_message) diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index 066c7a369d..d53334c1c9 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -193,10 +193,19 @@ def __or__(self, other: ActionSettings) -> ActionSettings: return result async def action(self, ctx: FilterContext) -> None: - """Execute the action of every action entry stored.""" + """Execute the action of every action entry stored, as well as any additional actions in the context.""" for entry in self.values(): await entry.action(ctx) + _i = len(ctx.additional_actions) + try: + for _i, action in enumerate(ctx.additional_actions): + await action + except Exception: + for action in ctx.additional_actions[_i+1:]: + action.close() + raise + def fallback_to(self, fallback: ActionSettings) -> ActionSettings: """Fill in missing entries from `fallback`.""" new_actions = self.copy() diff --git a/bot/exts/filtering/_settings_types/actions/delete_messages.py b/bot/exts/filtering/_settings_types/actions/delete_messages.py index d1ddf82412..1770c29ec6 100644 --- a/bot/exts/filtering/_settings_types/actions/delete_messages.py +++ b/bot/exts/filtering/_settings_types/actions/delete_messages.py @@ -1,4 +1,3 @@ -from contextlib import suppress from typing import ClassVar from discord.errors import NotFound @@ -25,9 +24,12 @@ async def action(self, ctx: FilterContext) -> None: if not ctx.message.guild: return - with suppress(NotFound): + try: await ctx.message.delete() - ctx.action_descriptions.append("deleted") + except NotFound: + ctx.action_descriptions.append("failed to delete") + else: + ctx.action_descriptions.append("deleted") def __or__(self, other: ActionEntry): """Combines two actions of the same type. Each type of action is executed once per filter.""" diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 7835a7d0bd..922101d6da 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -149,7 +149,7 @@ async def action(self, ctx: FilterContext) -> None: await ctx.author.send(dm_content, embed=dm_embed) ctx.action_descriptions.append("notified") except Forbidden: - ctx.action_descriptions.append("notified (failed)") + ctx.action_descriptions.append("failed to notify") if self.infraction_type is not None: await self.infraction_type.invoke( From 1d628b533c68f87ede4129516bc7e239459e85f0 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 23 Oct 2022 21:26:38 +0300 Subject: [PATCH 055/132] Add rich embed filter --- bot/exts/filtering/_filter_context.py | 5 ++- .../filtering/_filters/unique/rich_embed.py | 43 +++++++++++++++++++ bot/exts/filtering/_filters/unique/webhook.py | 23 ++++++---- bot/exts/filtering/_settings.py | 10 +---- 4 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 bot/exts/filtering/_filters/unique/rich_embed.py diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index da7ba0c77c..bcbafe3930 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Coroutine +from collections.abc import Callable, Coroutine from dataclasses import dataclass, field, replace from enum import Enum, auto from typing import Optional, Union @@ -35,7 +35,8 @@ class FilterContext: action_descriptions: list = field(default_factory=list) # What actions were taken matches: list = field(default_factory=list) # What exactly was found notification_domain: str = field(default_factory=str) # A domain to send the user for context - additional_actions: list[Coroutine] = field(default_factory=list) # Additional actions to perform + # Additional actions to perform + additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list) def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" diff --git a/bot/exts/filtering/_filters/unique/rich_embed.py b/bot/exts/filtering/_filters/unique/rich_embed.py new file mode 100644 index 0000000000..75f578d3ef --- /dev/null +++ b/bot/exts/filtering/_filters/unique/rich_embed.py @@ -0,0 +1,43 @@ +import re + +from botcore.utils.logging import get_logger + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.utils.helpers import remove_subdomain_from_url + +log = get_logger(__name__) + +URL_RE = re.compile(r"(https?://\S+)", flags=re.IGNORECASE) + + +class RichEmbedFilter(UniqueFilter): + """Filter messages which contain rich embeds not auto-generated from a URL.""" + + name = "rich_embed" + events = (Event.MESSAGE, Event.MESSAGE_EDIT) + + def triggered_on(self, ctx: FilterContext) -> bool: + """Determine if `msg` contains any rich embeds not auto-generated from a URL.""" + if ctx.message.embeds: + for embed in ctx.message.embeds: + if embed.type == "rich": + urls = URL_RE.findall(ctx.content) + final_urls = set(urls) + # This is due to the way discord renders relative urls in Embeds + # if the following url is sent: https://mobile.twitter.com/something + # Discord renders it as https://twitter.com/something + for url in urls: + final_urls.add(remove_subdomain_from_url(url)) + if not embed.url or embed.url not in final_urls: + # If `embed.url` does not exist or if `embed.url` is not part of the content + # of the message, it's unlikely to be an auto-generated embed by Discord. + ctx.alert_embeds.extend(ctx.message.embeds) + return True + else: + log.trace( + "Found a rich embed sent by a regular user account, " + "but it was likely just an automatic URL embed." + ) + + return False diff --git a/bot/exts/filtering/_filters/unique/webhook.py b/bot/exts/filtering/_filters/unique/webhook.py index 10f27a922d..ee6b7135e6 100644 --- a/bot/exts/filtering/_filters/unique/webhook.py +++ b/bot/exts/filtering/_filters/unique/webhook.py @@ -1,4 +1,5 @@ import re +from collections.abc import Callable, Coroutine from botcore.utils.logging import get_logger @@ -41,18 +42,22 @@ def triggered_on(self, ctx: FilterContext) -> bool: for i, match in enumerate(matches, start=1): extra = "" if len(matches) == 1 else f" ({i})" # Queue the webhook for deletion. - ctx.additional_actions.append(self._delete_webhook(ctx, match[0], extra)) + ctx.additional_actions.append(self._delete_webhook_wrapper(match[0], extra)) # Don't show the full webhook in places such as the mod alert. ctx.content = ctx.content.replace(match[0], match[1] + "xxx") return True @staticmethod - async def _delete_webhook(ctx: FilterContext, webhook_url: str, extra_message: str) -> None: - """Delete the given webhook and update the filter context.""" - async with bot.instance.http_session.delete(webhook_url) as resp: - # The Discord API Returns a 204 NO CONTENT response on success. - if resp.status == 204: - ctx.action_descriptions.append("webhook deleted" + extra_message) - else: - ctx.action_descriptions.append("failed to delete webhook" + extra_message) + def _delete_webhook_wrapper(webhook_url: str, extra_message: str) -> Callable[[FilterContext], Coroutine]: + """Create the action to perform when a webhook should be deleted.""" + async def _delete_webhook(ctx: FilterContext) -> None: + """Delete the given webhook and update the filter context.""" + async with bot.instance.http_session.delete(webhook_url) as resp: + # The Discord API Returns a 204 NO CONTENT response on success. + if resp.status == 204: + ctx.action_descriptions.append("webhook deleted" + extra_message) + else: + ctx.action_descriptions.append("failed to delete webhook" + extra_message) + + return _delete_webhook diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index d53334c1c9..85a6f3d2b5 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -197,14 +197,8 @@ async def action(self, ctx: FilterContext) -> None: for entry in self.values(): await entry.action(ctx) - _i = len(ctx.additional_actions) - try: - for _i, action in enumerate(ctx.additional_actions): - await action - except Exception: - for action in ctx.additional_actions[_i+1:]: - action.close() - raise + for action in ctx.additional_actions: + await action(ctx) def fallback_to(self, fallback: ActionSettings) -> ActionSettings: """Fill in missing entries from `fallback`.""" From 08208df37b036cfc76eea0fd276aba61b602951c Mon Sep 17 00:00:00 2001 From: mbaruh Date: Wed, 26 Oct 2022 23:39:44 +0300 Subject: [PATCH 056/132] Add Discord token filter Also fix a bug with the cog trying to serialize a set when trying to modify the DB with no UI. Also fix a bug with the domain setting description having a mismatching name. --- .../filtering/_filter_lists/filter_list.py | 7 +- bot/exts/filtering/_filters/domain.py | 2 +- .../_filters/unique/discord_token.py | 217 ++++++++++++++++++ .../filtering/_settings_types/actions/ping.py | 32 +-- bot/exts/filtering/_ui/ui.py | 2 +- bot/exts/filtering/_utils.py | 30 +++ 6 files changed, 255 insertions(+), 35 deletions(-) create mode 100644 bot/exts/filtering/_filters/unique/discord_token.py diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 55204335b7..50793b085b 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -202,7 +202,7 @@ class SubscribingAtomicList(AtomicList): Each unique filter is subscribed to a subset of events to respond to. """ - subscriptions: defaultdict[Event, list[Filter]] = dataclasses.field(default_factory=lambda: defaultdict(list)) + subscriptions: defaultdict[Event, list[int]] = dataclasses.field(default_factory=lambda: defaultdict(list)) def subscribe(self, filter_: UniqueFilter, *events: Event) -> None: """ @@ -213,8 +213,9 @@ def subscribe(self, filter_: UniqueFilter, *events: Event) -> None: """ for event in events: if filter_ not in self.subscriptions[event]: - self.subscriptions[event].append(filter_) + self.subscriptions[event].append(filter_.id) def filter_list_result(self, ctx: FilterContext) -> list[Filter]: """Sift through the list of filters, and return only the ones which apply to the given context.""" - return self._create_filter_list_result(ctx, self.defaults, self.subscriptions[ctx.event]) + event_filters = [self.filters[id_] for id_ in self.subscriptions[ctx.event]] + return self._create_filter_list_result(ctx, self.defaults, event_filters) diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 4976198cd9..e22cafbb76 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -15,7 +15,7 @@ class ExtraDomainSettings(BaseModel): """Extra settings for how domains should be matched in a message.""" - exact_description: ClassVar[str] = ( + subdomains_description: ClassVar[str] = ( "A boolean. If True, will will only trigger for subdomains and subpaths, and not for the domain itself." ) diff --git a/bot/exts/filtering/_filters/unique/discord_token.py b/bot/exts/filtering/_filters/unique/discord_token.py new file mode 100644 index 0000000000..571a8a9b16 --- /dev/null +++ b/bot/exts/filtering/_filters/unique/discord_token.py @@ -0,0 +1,217 @@ +import base64 +import re +from collections.abc import Callable, Coroutine +from typing import ClassVar, NamedTuple + +import discord +from botcore.utils.logging import get_logger +from botcore.utils.members import get_or_fetch_member +from pydantic import BaseModel, Field + +import bot +from bot import constants, utils +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._utils import resolve_mention +from bot.exts.moderation.modlog import ModLog +from bot.utils.messages import format_user + +log = get_logger(__name__) + + +LOG_MESSAGE = ( + "Censored a seemingly valid token sent by {author} in {channel}. " + "Token was: `{user_id}.{timestamp}.{hmac}`." +) +UNKNOWN_USER_LOG_MESSAGE = "Decoded user ID: `{user_id}` (Not present in server)." +KNOWN_USER_LOG_MESSAGE = ( + "Decoded user ID: `{user_id}` **(Present in server)**.\n" + "This matches `{user_name}` and means this is likely a valid **{kind}** token." +) +DISCORD_EPOCH = 1_420_070_400 +TOKEN_EPOCH = 1_293_840_000 + +# Three parts delimited by dots: user ID, creation timestamp, HMAC. +# The HMAC isn't parsed further, but it's in the regex to ensure it at least exists in the string. +# Each part only matches base64 URL-safe characters. +# These regexes were taken from discord-developers, which are used by the client itself. +TOKEN_RE = re.compile(r"([\w-]{10,})\.([\w-]{5,})\.([\w-]{10,})") + + +class ExtraDiscordTokenSettings(BaseModel): + """Extra settings for who should be pinged when a Discord token is detected.""" + + pings_for_bot_description: ClassVar[str] = "A sequence. Who should be pinged if the token found belongs to a bot." + pings_for_user_description: ClassVar[str] = "A sequence. Who should be pinged if the token found belongs to a user." + + pings_for_bot: set[str] = Field(default_factory=set) + pings_for_user: set[str] = Field(default_factory=lambda: {"Moderators"}) + + +class Token(NamedTuple): + """A Discord Bot token.""" + + user_id: str + timestamp: str + hmac: str + + +class DiscordTokenFilter(UniqueFilter): + """Scans messages for potential discord client tokens and removes them.""" + + name = "discord_token" + events = (Event.MESSAGE, Event.MESSAGE_EDIT) + extra_fields_type = ExtraDiscordTokenSettings + + @property + def mod_log(self) -> ModLog | None: + """Get currently loaded ModLog cog instance.""" + return bot.instance.get_cog("ModLog") + + def triggered_on(self, ctx: FilterContext) -> bool: + """Return whether the message contains Discord client tokens.""" + found_token = self.find_token_in_message(ctx.content) + if not found_token: + return False + + if mod_log := self.mod_log: + mod_log.ignore(constants.Event.message_delete, ctx.message.id) + ctx.content = ctx.content.replace(found_token.hmac, self.censor_hmac(found_token.hmac)) + ctx.additional_actions.append(self._create_token_alert_embed_wrapper(found_token)) + return True + + def _create_token_alert_embed_wrapper(self, found_token: Token) -> Callable[[FilterContext], Coroutine]: + """Create the action to perform when an alert should be sent for a message containing a Discord token.""" + async def _create_token_alert_embed(ctx: FilterContext) -> None: + """Add an alert embed to the context with info about the token sent.""" + userid_message, is_user = await self.format_userid_log_message(ctx.message, found_token) + log_message = self.format_log_message(ctx.message, found_token) + log.debug(log_message) + + if is_user: + mentions = map(resolve_mention, self.extra_fields.pings_for_user) + color = discord.Colour.red() + else: + mentions = map(resolve_mention, self.extra_fields.pings_for_bot) + color = discord.Colour.blue() + unmentioned = [mention for mention in mentions if mention not in ctx.alert_content] + if unmentioned: + ctx.alert_content = f"{' '.join(unmentioned)} {ctx.alert_content}" + ctx.alert_embeds.append(discord.Embed(colour=color, description=userid_message)) + + return _create_token_alert_embed + + @classmethod + async def format_userid_log_message(cls, msg: discord.Message, token: Token) -> tuple[str, bool]: + """ + Format the portion of the log message that includes details about the detected user ID. + + If the user is resolved to a member, the format includes the user ID, name, and the + kind of user detected. + If it is resolved to a user or a member, and it is not a bot, also return True. + Returns a tuple of (log_message, is_user) + """ + user_id = cls.extract_user_id(token.user_id) + user = await get_or_fetch_member(msg.guild, user_id) + + if user: + return KNOWN_USER_LOG_MESSAGE.format( + user_id=user_id, + user_name=str(user), + kind="BOT" if user.bot else "USER", + ), True + else: + return UNKNOWN_USER_LOG_MESSAGE.format(user_id=user_id), False + + @staticmethod + def censor_hmac(hmac: str) -> str: + """Return a censored version of the hmac.""" + return 'x' * (len(hmac) - 3) + hmac[-3:] + + @classmethod + def format_log_message(cls, msg: discord.Message, token: Token) -> str: + """Return the generic portion of the log message to send for `token` being censored in `msg`.""" + return LOG_MESSAGE.format( + author=format_user(msg.author), + channel=msg.channel.mention, + user_id=token.user_id, + timestamp=token.timestamp, + hmac=cls.censor_hmac(token.hmac), + ) + + @classmethod + def find_token_in_message(cls, content: str) -> Token | None: + """Return a seemingly valid token found in `msg` or `None` if no token is found.""" + # Use finditer rather than search to guard against method calls prematurely returning the + # token check (e.g. `message.channel.send` also matches our token pattern) + for match in TOKEN_RE.finditer(content): + token = Token(*match.groups()) + if ( + (cls.extract_user_id(token.user_id) is not None) + and cls.is_valid_timestamp(token.timestamp) + and cls.is_maybe_valid_hmac(token.hmac) + ): + # Short-circuit on first match + return token + + # No matching substring + return None + + @staticmethod + def extract_user_id(b64_content: str) -> int | None: + """Return a user ID integer from part of a potential token, or None if it couldn't be decoded.""" + b64_content = utils.pad_base64(b64_content) + + try: + decoded_bytes = base64.urlsafe_b64decode(b64_content) + string = decoded_bytes.decode('utf-8') + if not (string.isascii() and string.isdigit()): + # This case triggers if there are fancy unicode digits in the base64 encoding, + # that means it's not a valid user id. + return None + return int(string) + except ValueError: + return None + + @staticmethod + def is_valid_timestamp(b64_content: str) -> bool: + """ + Return True if `b64_content` decodes to a valid timestamp. + + If the timestamp is greater than the Discord epoch, it's probably valid. + See: https://i.imgur.com/7WdehGn.png + """ + b64_content = utils.pad_base64(b64_content) + + try: + decoded_bytes = base64.urlsafe_b64decode(b64_content) + timestamp = int.from_bytes(decoded_bytes, byteorder="big") + except ValueError as e: + log.debug(f"Failed to decode token timestamp '{b64_content}': {e}") + return False + + # Seems like newer tokens don't need the epoch added, but add anyway since an upper bound + # is not checked. + if timestamp + TOKEN_EPOCH >= DISCORD_EPOCH: + return True + else: + log.debug(f"Invalid token timestamp '{b64_content}': smaller than Discord epoch") + return False + + @staticmethod + def is_maybe_valid_hmac(b64_content: str) -> bool: + """ + Determine if a given HMAC portion of a token is potentially valid. + + If the HMAC has 3 or fewer characters, it's probably a dummy value like "xxxxxxxxxx", + and thus the token can probably be skipped. + """ + unique = len(set(b64_content.lower())) + if unique <= 3: + log.debug( + f"Considering the HMAC {b64_content} a dummy because it has {unique}" + " case-insensitively unique characters" + ) + return False + else: + return True diff --git a/bot/exts/filtering/_settings_types/actions/ping.py b/bot/exts/filtering/_settings_types/actions/ping.py index faac8f4b9c..5597bdd599 100644 --- a/bot/exts/filtering/_settings_types/actions/ping.py +++ b/bot/exts/filtering/_settings_types/actions/ping.py @@ -1,12 +1,10 @@ -from functools import cache from typing import ClassVar from pydantic import validator -import bot -from bot.constants import Guild from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +from bot.exts.filtering._utils import resolve_mention class Ping(ActionEntry): @@ -38,7 +36,7 @@ def init_sequence_if_none(cls, pings: list[str] | None) -> list[str]: async def action(self, ctx: FilterContext) -> None: """Add the stored pings to the alert message content.""" mentions = self.guild_pings if ctx.channel.guild else self.dm_pings - new_content = " ".join([self._resolve_mention(mention) for mention in mentions]) + new_content = " ".join([resolve_mention(mention) for mention in mentions]) ctx.alert_content = f"{new_content} {ctx.alert_content}" def __or__(self, other: ActionEntry): @@ -47,29 +45,3 @@ def __or__(self, other: ActionEntry): return NotImplemented return Ping(guild_pings=self.guild_pings | other.guild_pings, dm_pings=self.dm_pings | other.dm_pings) - - @staticmethod - @cache - def _resolve_mention(mention: str) -> str: - """Return the appropriate formatting for the formatting, be it a literal, a user ID, or a role ID.""" - guild = bot.instance.get_guild(Guild.id) - if mention in ("here", "everyone"): - return f"@{mention}" - try: - mention = int(mention) # It's an ID. - except ValueError: - pass - else: - if any(mention == role.id for role in guild.roles): - return f"<@&{mention}>" - else: - return f"<@{mention}>" - - # It's a name - for role in guild.roles: - if role.name == mention: - return role.mention - for member in guild.members: - if str(member) == mention: - return member.mention - return mention diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index c506db1fe5..6a261bc46e 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -74,7 +74,7 @@ def parse_value(value: str, type_: type[T]) -> T: if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias type_ = type_.__origin__ if type_ in (tuple, list, set): - return type_(value.split(",")) + return list(value.split(",")) if type_ is bool: return value.lower() == "true" or value == "1" if isinstance(type_, EnumMeta): diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index a38fa22e41..d5dfbfc830 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -4,10 +4,14 @@ import pkgutil from abc import ABC, abstractmethod from collections import defaultdict +from functools import cache from typing import Any, Iterable, TypeVar, Union import regex +import bot +from bot.constants import Guild + VARIATION_SELECTORS = r"\uFE00-\uFE0F\U000E0100-\U000E01EF" INVISIBLE_RE = regex.compile(rf"[{VARIATION_SELECTORS}\p{{UNASSIGNED}}\p{{FORMAT}}\p{{CONTROL}}--\s]", regex.V1) ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIATION_SELECTORS}]]", regex.V1) @@ -69,6 +73,32 @@ def to_serializable(item: Any) -> Union[bool, int, float, str, list, dict, None] return str(item) +@cache +def resolve_mention(mention: str) -> str: + """Return the appropriate formatting for the mention, be it a literal, a user ID, or a role ID.""" + guild = bot.instance.get_guild(Guild.id) + if mention in ("here", "everyone"): + return f"@{mention}" + try: + mention = int(mention) # It's an ID. + except ValueError: + pass + else: + if any(mention == role.id for role in guild.roles): + return f"<@&{mention}>" + else: + return f"<@{mention}>" + + # It's a name + for role in guild.roles: + if role.name == mention: + return role.mention + for member in guild.members: + if str(member) == mention: + return member.mention + return mention + + def repr_equals(override: Any, default: Any) -> bool: """Return whether the override and the default have the same representation.""" if override is None: # It's not an override From c756c52fc0fe2705309e532fdcfe4f669e284745 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 27 Oct 2022 23:19:21 +0300 Subject: [PATCH 057/132] Suppress exceptions while actioning filters Filtering should not stop even if one type of action raised an exception. For example, if deleting the message raised somehow, it should still try to infract the user. Exceptions raised in actions are logged instead. Additionally adds a fix to the way actions are merged. --- bot/exts/filtering/_filter_lists/filter_list.py | 14 +++++++++----- bot/exts/filtering/_settings.py | 13 +++++++++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 50793b085b..0bb0dc7f8e 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -113,12 +113,16 @@ def merge_actions(self, filters: list[Filter]) -> ActionSettings | None: If `merge_default` is True, include it in the merge instead of using it as a fallback. """ - try: - result = reduce(or_, (filter_.actions for filter_ in filters if filter_.actions)) - except TypeError: # The sequence fed to reduce is empty. + if not filters: # Nothing to action. return None - - return result.fallback_to(self.defaults.actions) + try: + return reduce( + or_, (filter_.actions or self.defaults.actions for filter_ in filters) + ).fallback_to(self.defaults.actions) + except TypeError: + # The sequence fed to reduce is empty, meaning none of the filters have actions, + # meaning they all use the defaults. + return self.defaults.actions @staticmethod def format_messages(triggers: list[Filter], *, expand_single_filter: bool = True) -> list[str]: diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index 85a6f3d2b5..a752918a62 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -1,6 +1,7 @@ from __future__ import annotations import operator +import traceback from abc import abstractmethod from functools import reduce from typing import Any, NamedTuple, Optional, TypeVar @@ -195,10 +196,18 @@ def __or__(self, other: ActionSettings) -> ActionSettings: async def action(self, ctx: FilterContext) -> None: """Execute the action of every action entry stored, as well as any additional actions in the context.""" for entry in self.values(): - await entry.action(ctx) + try: + await entry.action(ctx) + # Filtering should not stop even if one type of action raised an exception. + # For example, if deleting the message raised somehow, it should still try to infract the user. + except Exception: + log.exception(traceback.format_exc()) for action in ctx.additional_actions: - await action(ctx) + try: + await action(ctx) + except Exception: + log.exception(traceback.format_exc()) def fallback_to(self, fallback: ActionSettings) -> ActionSettings: """Fill in missing entries from `fallback`.""" From 38281298a199abe44ec2143386caee6c728446aa Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 27 Oct 2022 23:29:41 +0300 Subject: [PATCH 058/132] Convert redundant filter setting group to command --- bot/exts/filtering/filtering.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 837cd45c12..d2b07de4e0 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -459,21 +459,17 @@ async def delete_list() -> None: view=DeleteConfirmationView(ctx.author, delete_list) ) - @filter.group(aliases=("settings",)) - async def setting(self, ctx: Context) -> None: - """Group for settings-related commands.""" - if not ctx.invoked_subcommand: - await ctx.send_help(ctx.command) - - @setting.command(name="describe", aliases=("explain", "manual")) - async def s_describe(self, ctx: Context, setting_name: Optional[str]) -> None: + @filter.command(aliases=("settings",)) + async def setting(self, ctx: Context, setting_name: str | None) -> None: """Show a description of the specified setting, or a list of possible settings if no name is specified.""" if not setting_name: - settings_list = list(self.loaded_settings) + settings_list = [f"» {setting_name}" for setting_name in self.loaded_settings] for filter_name, filter_settings in self.loaded_filter_settings.items(): - settings_list.extend(f"{filter_name}/{setting}" for setting in filter_settings) - embed = Embed(description="\n".join(settings_list)) + settings_list.extend(f"» {filter_name}/{setting}" for setting in filter_settings) + embed = Embed(colour=Colour.blue()) embed.set_author(name="List of setting names") + await LinePaginator.paginate(settings_list, ctx, embed, max_lines=10, empty=False) + else: # The setting is either in a SettingsEntry subclass, or a pydantic model. setting_data = self.loaded_settings.get(setting_name) @@ -488,10 +484,9 @@ async def s_describe(self, ctx: Context, setting_name: Optional[str]) -> None: if description is None: await ctx.send(f":x: There's no setting type named {setting_name!r}.") return - embed = Embed(description=description) + embed = Embed(colour=Colour.blue(), description=description) embed.set_author(name=f"Description of the {setting_name} setting") - embed.colour = Colour.blue() - await ctx.send(embed=embed) + await ctx.send(embed=embed) @filter.command(name="match") async def f_match(self, ctx: Context, message: Message | None, *, string: str | None) -> None: From 46cade75bb0ca3f60f3a4375d0f3be12b122290e Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 27 Oct 2022 23:37:13 +0300 Subject: [PATCH 059/132] Use command prefix instead of hardcoded ! --- bot/exts/filtering/filtering.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index d2b07de4e0..44eaa5ea72 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -15,6 +15,7 @@ import bot import bot.exts.filtering._ui.filter as filters_ui +from bot import constants from bot.bot import Bot from bot.constants import Channels, Colours, MODERATION_ROLES, Roles, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext @@ -294,7 +295,8 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: embed.set_author(name=f"Filter #{id_} - " + f"{filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " - f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + f"To view all defaults of the list, " + f"run `{constants.Bot.prefix}filterlist describe {list_type.name} {filter_list.name}`." )) await ctx.send(embed=embed) @@ -421,7 +423,8 @@ async def f_edit( name=f"Filter #{filter_id} - {filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " - f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + f"To view all defaults of the list, " + f"run `{constants.Bot.prefix}filterlist describe {list_type.name} {filter_list.name}`." )) view = filters_ui.FilterEditView( @@ -888,7 +891,8 @@ async def _add_filter( name=f"New Filter - {filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " - f"To view all defaults of the list, run `!filterlist describe {list_type.name} {filter_list.name}`." + f"To view all defaults of the list, " + f"run `{constants.Bot.prefix}filterlist describe {list_type.name} {filter_list.name}`." )) view = filters_ui.FilterEditView( From c13d7619af4d65ad73c40122660ab9a9361ca751 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 28 Oct 2022 01:26:00 +0300 Subject: [PATCH 060/132] Handle context message possibly being None --- bot/exts/filtering/_filter_context.py | 7 +++---- .../filtering/_filters/unique/discord_token.py | 18 ++++++++++-------- .../filtering/_filters/unique/rich_embed.py | 6 +++--- bot/exts/filtering/_filters/unique/webhook.py | 2 +- .../_settings_types/actions/delete_messages.py | 4 ++-- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index bcbafe3930..22950d5df4 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -3,7 +3,6 @@ from collections.abc import Callable, Coroutine from dataclasses import dataclass, field, replace from enum import Enum, auto -from typing import Optional, Union from discord import DMChannel, Member, Message, TextChannel, Thread, User @@ -22,9 +21,9 @@ class FilterContext: # Input context event: Event # The type of event author: User | Member | None # Who triggered the event - channel: Union[TextChannel, Thread, DMChannel] # The channel involved - content: Union[str, set] # What actually needs filtering - message: Optional[Message] # The message involved + channel: TextChannel | Thread | DMChannel # The channel involved + content: str | set # What actually needs filtering + message: Message | None # The message involved embeds: list = field(default_factory=list) # Any embeds involved # Output context dm_content: str = field(default_factory=str) # The content to DM the invoker diff --git a/bot/exts/filtering/_filters/unique/discord_token.py b/bot/exts/filtering/_filters/unique/discord_token.py index 571a8a9b16..7fdb800df6 100644 --- a/bot/exts/filtering/_filters/unique/discord_token.py +++ b/bot/exts/filtering/_filters/unique/discord_token.py @@ -10,6 +10,7 @@ import bot from bot import constants, utils +from bot.constants import Guild from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filters.filter import UniqueFilter from bot.exts.filtering._utils import resolve_mention @@ -74,7 +75,7 @@ def triggered_on(self, ctx: FilterContext) -> bool: if not found_token: return False - if mod_log := self.mod_log: + if ctx.message and (mod_log := self.mod_log): mod_log.ignore(constants.Event.message_delete, ctx.message.id) ctx.content = ctx.content.replace(found_token.hmac, self.censor_hmac(found_token.hmac)) ctx.additional_actions.append(self._create_token_alert_embed_wrapper(found_token)) @@ -84,8 +85,8 @@ def _create_token_alert_embed_wrapper(self, found_token: Token) -> Callable[[Fil """Create the action to perform when an alert should be sent for a message containing a Discord token.""" async def _create_token_alert_embed(ctx: FilterContext) -> None: """Add an alert embed to the context with info about the token sent.""" - userid_message, is_user = await self.format_userid_log_message(ctx.message, found_token) - log_message = self.format_log_message(ctx.message, found_token) + userid_message, is_user = await self.format_userid_log_message(found_token) + log_message = self.format_log_message(ctx.author, ctx.channel, found_token) log.debug(log_message) if is_user: @@ -102,7 +103,7 @@ async def _create_token_alert_embed(ctx: FilterContext) -> None: return _create_token_alert_embed @classmethod - async def format_userid_log_message(cls, msg: discord.Message, token: Token) -> tuple[str, bool]: + async def format_userid_log_message(cls, token: Token) -> tuple[str, bool]: """ Format the portion of the log message that includes details about the detected user ID. @@ -112,7 +113,8 @@ async def format_userid_log_message(cls, msg: discord.Message, token: Token) -> Returns a tuple of (log_message, is_user) """ user_id = cls.extract_user_id(token.user_id) - user = await get_or_fetch_member(msg.guild, user_id) + guild = bot.instance.get_guild(Guild.id) + user = await get_or_fetch_member(guild, user_id) if user: return KNOWN_USER_LOG_MESSAGE.format( @@ -129,11 +131,11 @@ def censor_hmac(hmac: str) -> str: return 'x' * (len(hmac) - 3) + hmac[-3:] @classmethod - def format_log_message(cls, msg: discord.Message, token: Token) -> str: + def format_log_message(cls, author: discord.User, channel: discord.abc.GuildChannel, token: Token) -> str: """Return the generic portion of the log message to send for `token` being censored in `msg`.""" return LOG_MESSAGE.format( - author=format_user(msg.author), - channel=msg.channel.mention, + author=format_user(author), + channel=channel.mention, user_id=token.user_id, timestamp=token.timestamp, hmac=cls.censor_hmac(token.hmac), diff --git a/bot/exts/filtering/_filters/unique/rich_embed.py b/bot/exts/filtering/_filters/unique/rich_embed.py index 75f578d3ef..a0d9e263f0 100644 --- a/bot/exts/filtering/_filters/unique/rich_embed.py +++ b/bot/exts/filtering/_filters/unique/rich_embed.py @@ -19,8 +19,8 @@ class RichEmbedFilter(UniqueFilter): def triggered_on(self, ctx: FilterContext) -> bool: """Determine if `msg` contains any rich embeds not auto-generated from a URL.""" - if ctx.message.embeds: - for embed in ctx.message.embeds: + if ctx.embeds: + for embed in ctx.embeds: if embed.type == "rich": urls = URL_RE.findall(ctx.content) final_urls = set(urls) @@ -32,7 +32,7 @@ def triggered_on(self, ctx: FilterContext) -> bool: if not embed.url or embed.url not in final_urls: # If `embed.url` does not exist or if `embed.url` is not part of the content # of the message, it's unlikely to be an auto-generated embed by Discord. - ctx.alert_embeds.extend(ctx.message.embeds) + ctx.alert_embeds.extend(ctx.embeds) return True else: log.trace( diff --git a/bot/exts/filtering/_filters/unique/webhook.py b/bot/exts/filtering/_filters/unique/webhook.py index ee6b7135e6..b9d98db350 100644 --- a/bot/exts/filtering/_filters/unique/webhook.py +++ b/bot/exts/filtering/_filters/unique/webhook.py @@ -36,7 +36,7 @@ def triggered_on(self, ctx: FilterContext) -> bool: return False # Don't log this. - if mod_log := self.mod_log: + if ctx.message and (mod_log := self.mod_log): mod_log.ignore(constants.Event.message_delete, ctx.message.id) for i, match in enumerate(matches, start=1): diff --git a/bot/exts/filtering/_settings_types/actions/delete_messages.py b/bot/exts/filtering/_settings_types/actions/delete_messages.py index 1770c29ec6..2f851ef048 100644 --- a/bot/exts/filtering/_settings_types/actions/delete_messages.py +++ b/bot/exts/filtering/_settings_types/actions/delete_messages.py @@ -2,7 +2,7 @@ from discord.errors import NotFound -from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry @@ -18,7 +18,7 @@ class DeleteMessages(ActionEntry): async def action(self, ctx: FilterContext) -> None: """Delete the context message(s).""" - if not self.delete_messages or ctx.event not in (Event.MESSAGE, Event.MESSAGE_EDIT): + if not self.delete_messages or not ctx.message: return if not ctx.message.guild: From 3fc53356b5746533d76d82846b8fd9b9c649eac5 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 30 Oct 2022 22:51:15 +0200 Subject: [PATCH 061/132] Add antispam filter list and duplicates filter Adds the antispam filterlist, which dispatches a set of messages sent in the last X seconds to its filters. Unlike other filter lists, it doesn't just return the actions to be taken, but rather splits them in two: any actions unrelated to alerting mods are returned as usual, while actions related to alerting are used in a delayed coroutine, from which the alert is sent. - Fixes `infraction_channel` not using the context channel when it's None. - Moves the `upload_log` function outside the ModLog cog, as it only needed it for the bot instance. - Any auto-deleted message with attachments will now have its attachments logged. --- bot/exts/filtering/_filter_context.py | 14 +- bot/exts/filtering/_filter_lists/antispam.py | 179 ++++++++++++++++++ .../filtering/_filter_lists/filter_list.py | 47 +++++ bot/exts/filtering/_filter_lists/unique.py | 41 +--- .../filtering/_filters/antispam/__init__.py | 9 + .../filtering/_filters/antispam/duplicates.py | 44 +++++ .../actions/delete_messages.py | 49 ++++- .../actions/infraction_and_notification.py | 25 ++- bot/exts/filtering/_ui/ui.py | 61 +++++- bot/exts/filtering/_utils.py | 2 +- bot/exts/filtering/filtering.py | 46 ++--- bot/exts/moderation/clean.py | 3 +- bot/exts/moderation/modlog.py | 51 +---- bot/utils/messages.py | 49 ++++- 14 files changed, 479 insertions(+), 141 deletions(-) create mode 100644 bot/exts/filtering/_filter_lists/antispam.py create mode 100644 bot/exts/filtering/_filters/antispam/__init__.py create mode 100644 bot/exts/filtering/_filters/antispam/duplicates.py diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 22950d5df4..3227b333ae 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -1,11 +1,15 @@ from __future__ import annotations -from collections.abc import Callable, Coroutine +import typing +from collections.abc import Callable, Coroutine, Iterable from dataclasses import dataclass, field, replace from enum import Enum, auto from discord import DMChannel, Member, Message, TextChannel, Thread, User +if typing.TYPE_CHECKING: + from bot.exts.filtering._filters.filter import Filter + class Event(Enum): """Types of events that can trigger filtering. Note this does not have to align with gateway event types.""" @@ -22,7 +26,7 @@ class FilterContext: event: Event # The type of event author: User | Member | None # Who triggered the event channel: TextChannel | Thread | DMChannel # The channel involved - content: str | set # What actually needs filtering + content: str | Iterable # What actually needs filtering message: Message | None # The message involved embeds: list = field(default_factory=list) # Any embeds involved # Output context @@ -32,10 +36,14 @@ class FilterContext: alert_content: str = field(default_factory=str) # The content of the alert alert_embeds: list = field(default_factory=list) # Any embeds to add to the alert action_descriptions: list = field(default_factory=list) # What actions were taken - matches: list = field(default_factory=list) # What exactly was found + matches: list[str] = field(default_factory=list) # What exactly was found notification_domain: str = field(default_factory=str) # A domain to send the user for context + filter_info: dict['Filter', str] = field(default_factory=dict) # Additional info from a filter. # Additional actions to perform additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list) + related_messages: set[Message] = field(default_factory=set) + related_channels: set[TextChannel | Thread | DMChannel] = field(default_factory=set) + attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py new file mode 100644 index 0000000000..2dab54ce62 --- /dev/null +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -0,0 +1,179 @@ +import asyncio +import typing +from collections.abc import Callable, Coroutine +from dataclasses import dataclass, field +from datetime import timedelta +from functools import reduce +from itertools import takewhile +from operator import add, or_ + +import arrow +from botcore.utils import scheduling +from botcore.utils.logging import get_logger +from discord import HTTPException, Member + +import bot +from bot.constants import Webhooks +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_lists.filter_list import ListType, SubscribingAtomicList, UniquesListBase +from bot.exts.filtering._filters.antispam import antispam_filter_types +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._settings import ActionSettings +from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction +from bot.exts.filtering._ui.ui import build_mod_alert + +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + +log = get_logger(__name__) + +ALERT_DELAY = 6 + + +class AntispamList(UniquesListBase): + """ + A list of anti-spam rules. + + Messages from the last X seconds is passed to each rule, which decide whether it triggers across those messages. + """ + + name = "antispam" + + def __init__(self, filtering_cog: 'Filtering'): + super().__init__(filtering_cog) + self.message_deletion_queue: dict[Member, DeletionContext] = dict() + + def get_filter_type(self, content: str) -> type[UniqueFilter] | None: + """Get a subclass of filter matching the filter list and the filter's content.""" + try: + return antispam_filter_types[content] + except KeyError: + if content not in self._already_warned: + log.warn(f"An antispam filter named {content} was supplied, but no matching implementation found.") + self._already_warned.add(content) + return None + + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" + if not ctx.message: + return None, [] + + sublist: SubscribingAtomicList = self[ListType.DENY] + potential_filters = [sublist.filters[id_] for id_ in sublist.subscriptions[ctx.event]] + max_interval = max(filter_.extra_fields.interval for filter_ in potential_filters) + + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=max_interval) + relevant_messages = list( + takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.filtering_cog.message_cache) + ) + new_ctx = ctx.replace(content=relevant_messages) + triggers = sublist.filter_list_result(new_ctx) + if not triggers: + return None, [] + + if ctx.author not in self.message_deletion_queue: + self.message_deletion_queue[ctx.author] = DeletionContext() + ctx.additional_actions.append(self._create_deletion_context_handler(ctx.author)) + ctx.related_channels |= {msg.channel for msg in ctx.related_messages} + else: # The additional messages found are already part of the deletion context + ctx.related_messages = set() + current_infraction = self.message_deletion_queue[ctx.author].current_infraction + self.message_deletion_queue[ctx.author].add(ctx, triggers) + + current_actions = sublist.merge_actions(triggers) if triggers else None + # Don't alert yet. + current_actions.pop("ping", None) + current_actions.pop("send_alert", None) + new_infraction = current_actions["infraction_and_notification"].copy() + # Smaller infraction value = higher in hierarchy. + if not current_infraction or new_infraction.infraction_type.value < current_infraction.value: + # Pick the first triggered filter for the reason, there's no good way to decide between them. + new_infraction.infraction_reason = f"{triggers[0].name} spam - {ctx.filter_info[triggers[0]]}" + current_actions["infraction_and_notification"] = new_infraction + self.message_deletion_queue[ctx.author].current_infraction = new_infraction.infraction_type + else: + current_actions.pop("infraction_and_notification", None) + + # Provide some message in case another filter list wants there to be an alert. + return current_actions, ["Handling spam event..."] + + def _create_deletion_context_handler(self, context_id: Member) -> Callable[[FilterContext], Coroutine]: + async def schedule_processing(ctx: FilterContext) -> None: + """ + Schedule a coroutine to process the deletion context. + + It cannot be awaited directly, as it waits ALERT_DELAY seconds, and actioning a filtering context depends on + all actions finishing. + + This is async and takes a context to adhere to the type of ctx.additional_actions. + """ + async def process_deletion_context() -> None: + """Processes the Deletion Context queue.""" + log.trace("Sleeping before processing message deletion queue.") + await asyncio.sleep(ALERT_DELAY) + + if context_id not in self.message_deletion_queue: + log.error(f"Started processing deletion queue for context `{context_id}`, but it was not found!") + return + + deletion_context = self.message_deletion_queue.pop(context_id) + await deletion_context.send_alert(self) + + scheduling.create_task(process_deletion_context()) + + return schedule_processing + + +@dataclass +class DeletionContext: + """Represents a Deletion Context for a single spam event.""" + + contexts: list[FilterContext] = field(default_factory=list) + rules: set[UniqueFilter] = field(default_factory=set) + current_infraction: Infraction | None = None + + def add(self, ctx: FilterContext, rules: list[UniqueFilter]) -> None: + """Adds new rule violation events to the deletion context.""" + self.contexts.append(ctx) + self.rules.update(rules) + + async def send_alert(self, antispam_list: AntispamList) -> None: + """Post the mod alert.""" + if not self.contexts or not self.rules: + return + try: + webhook = await bot.instance.fetch_webhook(Webhooks.filters) + except HTTPException: + return + + ctx, *other_contexts = self.contexts + new_ctx = FilterContext(ctx.event, ctx.author, ctx.channel, ctx.content, ctx.message) + new_ctx.action_descriptions = reduce( + add, (other_ctx.action_descriptions for other_ctx in other_contexts), ctx.action_descriptions + ) + # It shouldn't ever come to this, but just in case. + if descriptions_num := len(new_ctx.action_descriptions) > 20: + new_ctx.action_descriptions = new_ctx.action_descriptions[:20] + new_ctx.action_descriptions[-1] += f" (+{descriptions_num - 20} other actions)" + new_ctx.related_messages = reduce( + or_, (other_ctx.related_messages for other_ctx in other_contexts), ctx.related_messages + ) + new_ctx.related_channels = reduce( + or_, (other_ctx.related_channels for other_ctx in other_contexts), ctx.related_channels + ) + new_ctx.attachments = reduce(or_, (other_ctx.attachments for other_ctx in other_contexts), ctx.attachments) + + rules = list(self.rules) + actions = antispam_list[ListType.DENY].merge_actions(rules) + for action in list(actions): + if action not in ("ping", "send_alert"): + actions.pop(action, None) + await actions.action(new_ctx) + + messages = antispam_list[ListType.DENY].format_messages(rules) + embed = await build_mod_alert(new_ctx, {antispam_list: messages}) + if other_contexts: + embed.set_footer( + text="The list of actions taken includes actions from additional contexts after deletion began." + ) + await webhook.send(username="Anti-Spam", content=ctx.alert_content, embeds=[embed]) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 0bb0dc7f8e..f9db54a21f 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -17,6 +17,9 @@ from bot.exts.filtering._utils import FieldRequiring, past_tense from bot.log import get_logger +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + log = get_logger(__name__) @@ -223,3 +226,47 @@ def filter_list_result(self, ctx: FilterContext) -> list[Filter]: """Sift through the list of filters, and return only the ones which apply to the given context.""" event_filters = [self.filters[id_] for id_ in self.subscriptions[ctx.event]] return self._create_filter_list_result(ctx, self.defaults, event_filters) + + +class UniquesListBase(FilterList[UniqueFilter]): + """ + A list of unique filters. + + Unique filters are ones that should only be run once in a given context. + Each unique filter subscribes to a subset of events to respond to. + """ + + _already_warned = set() + + def __init__(self, filtering_cog: 'Filtering'): + super().__init__() + self.filtering_cog = filtering_cog + self.loaded_types: dict[str, type[UniqueFilter]] = {} + + def add_list(self, list_data: dict) -> SubscribingAtomicList: + """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" + actions, validations = create_settings(list_data["settings"], keep_empty=True) + list_type = ListType(list_data["list_type"]) + defaults = Defaults(actions, validations) + new_list = SubscribingAtomicList(list_data["id"], self.name, list_type, defaults, {}) + self[list_type] = new_list + + filters = {} + events = set() + for filter_data in list_data["filters"]: + new_filter = self._create_filter(filter_data, defaults) + if new_filter: + new_list.subscribe(new_filter, *new_filter.events) + filters[filter_data["id"]] = new_filter + self.loaded_types[new_filter.name] = type(new_filter) + events.update(new_filter.events) + + new_list.filters.update(filters) + if hasattr(self.filtering_cog, "subscribe"): # Subscribe the filter list to any new events found. + self.filtering_cog.subscribe(self, *events) + return new_list + + @property + def filter_types(self) -> set[type[UniqueFilter]]: + """Return the types of filters used by this list.""" + return set(self.loaded_types.values()) diff --git a/bot/exts/filtering/_filter_lists/unique.py b/bot/exts/filtering/_filter_lists/unique.py index 63caa7d36b..5204065f94 100644 --- a/bot/exts/filtering/_filter_lists/unique.py +++ b/bot/exts/filtering/_filter_lists/unique.py @@ -1,16 +1,15 @@ from botcore.utils.logging import get_logger -from discord.ext.commands import Cog from bot.exts.filtering._filter_context import FilterContext -from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType, SubscribingAtomicList +from bot.exts.filtering._filter_lists.filter_list import ListType, UniquesListBase from bot.exts.filtering._filters.filter import UniqueFilter from bot.exts.filtering._filters.unique import unique_filter_types -from bot.exts.filtering._settings import ActionSettings, Defaults, create_settings +from bot.exts.filtering._settings import ActionSettings log = get_logger(__name__) -class UniquesList(FilterList[UniqueFilter]): +class UniquesList(UniquesListBase): """ A list of unique filters. @@ -19,35 +18,6 @@ class UniquesList(FilterList[UniqueFilter]): """ name = "unique" - _already_warned = set() - - def __init__(self, filtering_cog: Cog): - super().__init__() - self.filtering_cog = filtering_cog # This is typed as a Cog to avoid a circular import. - self.loaded_types: dict[str, type[UniqueFilter]] = {} - - def add_list(self, list_data: dict) -> SubscribingAtomicList: - """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" - actions, validations = create_settings(list_data["settings"], keep_empty=True) - list_type = ListType(list_data["list_type"]) - defaults = Defaults(actions, validations) - new_list = SubscribingAtomicList(list_data["id"], self.name, list_type, defaults, {}) - self[list_type] = new_list - - filters = {} - events = set() - for filter_data in list_data["filters"]: - new_filter = self._create_filter(filter_data, defaults) - if new_filter: - new_list.subscribe(new_filter, *new_filter.events) - filters[filter_data["id"]] = new_filter - self.loaded_types[new_filter.name] = type(new_filter) - events.update(new_filter.events) - - new_list.filters.update(filters) - if hasattr(self.filtering_cog, "subscribe"): # Subscribe the filter list to any new events found. - self.filtering_cog.subscribe(self, *events) - return new_list def get_filter_type(self, content: str) -> type[UniqueFilter] | None: """Get a subclass of filter matching the filter list and the filter's content.""" @@ -59,11 +29,6 @@ def get_filter_type(self, content: str) -> type[UniqueFilter] | None: self._already_warned.add(content) return None - @property - def filter_types(self) -> set[type[UniqueFilter]]: - """Return the types of filters used by this list.""" - return set(self.loaded_types.values()) - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" triggers = self[ListType.DENY].filter_list_result(ctx) diff --git a/bot/exts/filtering/_filters/antispam/__init__.py b/bot/exts/filtering/_filters/antispam/__init__.py new file mode 100644 index 0000000000..637bcd4103 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname + +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._utils import subclasses_in_package + +antispam_filter_types = subclasses_in_package(dirname(__file__), f"{__name__}.", UniqueFilter) +antispam_filter_types = {filter_.name: filter_ for filter_ in antispam_filter_types} + +__all__ = [antispam_filter_types] diff --git a/bot/exts/filtering/_filters/antispam/duplicates.py b/bot/exts/filtering/_filters/antispam/duplicates.py new file mode 100644 index 0000000000..5df2bb5c0a --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/duplicates.py @@ -0,0 +1,44 @@ +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + + +class ExtraDuplicatesSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Number of duplicate messages required to trigger the filter." + + interval: int = 10 + threshold: int = 3 + + +class DuplicatesFilter(UniqueFilter): + """Detects duplicated messages sent by a single user.""" + + name = "duplicates" + events = (Event.MESSAGE,) + extra_fields_type = ExtraDuplicatesSettings + + def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + + detected_messages = { + msg for msg in relevant_messages + if msg.author == ctx.author and msg.content == ctx.message.content and msg.content + } + if len(detected_messages) > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {len(detected_messages)} duplicate messages" + return True + return False diff --git a/bot/exts/filtering/_settings_types/actions/delete_messages.py b/bot/exts/filtering/_settings_types/actions/delete_messages.py index 2f851ef048..19c0beb954 100644 --- a/bot/exts/filtering/_settings_types/actions/delete_messages.py +++ b/bot/exts/filtering/_settings_types/actions/delete_messages.py @@ -1,9 +1,24 @@ +from collections import defaultdict from typing import ClassVar -from discord.errors import NotFound +from botcore.utils import scheduling +from discord import Message +from discord.errors import HTTPException +from bot.constants import Channels from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +from bot.utils.messages import send_attachments + + +async def upload_messages_attachments(ctx: FilterContext, messages: list[Message]) -> None: + """Re-upload the messages' attachments for future logging.""" + if not messages: + return + destination = messages[0].guild.get_channel(Channels.attachment_log) + for message in messages: + if message.attachments and message.id not in ctx.attachments: + ctx.attachments[message.id] = await send_attachments(message, destination, link_large=False) class DeleteMessages(ActionEntry): @@ -24,12 +39,34 @@ async def action(self, ctx: FilterContext) -> None: if not ctx.message.guild: return - try: - await ctx.message.delete() - except NotFound: - ctx.action_descriptions.append("failed to delete") + channel_messages = defaultdict(set) # Duplicates will cause batch deletion to fail. + for message in {ctx.message} | ctx.related_messages: + channel_messages[message.channel].add(message) + + success = fail = 0 + deleted = list() + for channel, messages in channel_messages.items(): + try: + await channel.delete_messages(messages) + except HTTPException: + fail += len(messages) + else: + success += len(messages) + deleted.extend(messages) + scheduling.create_task(upload_messages_attachments(ctx, deleted)) + + if not fail: + if success == 1: + ctx.action_descriptions.append("deleted") + else: + ctx.action_descriptions.append("deleted all") + elif not success: + if fail == 1: + ctx.action_descriptions.append("failed to delete") + else: + ctx.action_descriptions.append("all failed to delete") else: - ctx.action_descriptions.append("deleted") + ctx.action_descriptions.append(f"{success} deleted, {fail} failed to delete") def __or__(self, other: ActionEntry): """Combines two actions of the same type. Each type of action is executed once per filter.""" diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 922101d6da..fb679855a0 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -5,6 +5,7 @@ import arrow import discord.abc +from botcore.utils.logging import get_logger from discord import Colour, Embed, Member, User from discord.errors import Forbidden from pydantic import validator @@ -14,6 +15,8 @@ from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +log = get_logger(__name__) + @dataclass class FakeContext: @@ -65,17 +68,12 @@ def __str__(self) -> str: async def invoke( self, user: Member | User, - channel: int | None, + channel: discord.abc.Messageable, + alerts_channel: discord.TextChannel, duration: float | None, reason: str | None ) -> None: """Invokes the command matching the infraction name.""" - alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) - if not channel: - channel = alerts_channel - else: - channel = bot_module.instance.get_channel(channel) - command_name = self.name.lower() command = bot_module.instance.get_command(command_name) if not command: @@ -108,7 +106,8 @@ class InfractionAndNotification(ActionEntry): "infraction_reason": "The reason delivered with the infraction.", "infraction_channel": ( "The channel ID in which to invoke the infraction (and send the confirmation message). " - "If blank, the infraction will be sent in the context channel." + "If blank, the infraction will be sent in the context channel. If the ID fails to resolve, it will default " + "to the mod-alerts channel." ), "dm_content": "The contents of a message to be DMed to the offending user.", "dm_embed": "The contents of the embed to be DMed to the offending user." @@ -152,8 +151,16 @@ async def action(self, ctx: FilterContext) -> None: ctx.action_descriptions.append("failed to notify") if self.infraction_type is not None: + alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) + if self.infraction_channel: + channel = bot_module.instance.get_channel(self.infraction_channel) + if not channel: + log.info(f"Could not find a channel with ID {self.infraction_channel}, infracting in mod-alerts.") + channel = alerts_channel + else: + channel = ctx.channel await self.infraction_type.invoke( - ctx.author, self.infraction_channel, self.infraction_duration, self.infraction_reason + ctx.author, channel, alerts_channel, self.infraction_duration, self.infraction_reason ) ctx.action_descriptions.append(self.infraction_type.name.lower()) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 6a261bc46e..9fc15410ef 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -13,6 +13,13 @@ from discord import Embed, Interaction from discord.ext.commands import Context from discord.ui.select import MISSING as SELECT_MISSING, SelectOption +from discord.utils import escape_markdown + +import bot +from bot.constants import Colours +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_lists import FilterList +from bot.utils.messages import format_channel, format_user, upload_log log = get_logger(__name__) @@ -31,7 +38,7 @@ MAX_MODAL_TITLE_LENGTH = 45 # Max number of items in a select MAX_SELECT_ITEMS = 25 -MAX_EMBED_DESCRIPTION = 4000 +MAX_EMBED_DESCRIPTION = 4080 SETTINGS_DELIMITER = re.compile(r"\s+(?=\S+=\S+)") SINGLE_SETTING_PATTERN = re.compile(r"[\w/]+=.+") @@ -42,6 +49,58 @@ T = TypeVar('T') +async def _build_alert_message_content(ctx: FilterContext, current_message_length: int) -> str: + """Build the content section of the alert.""" + # For multiple messages and those with attachments or excessive newlines, use the logs API + if any(( + ctx.related_messages, + len(ctx.attachments) > 0, + ctx.content.count('\n') > 15 + )): + url = await upload_log(ctx.related_messages, bot.instance.user.id, ctx.attachments) + alert_content = f"A complete log of the offending messages can be found [here]({url})" + else: + alert_content = escape_markdown(ctx.content) + remaining_chars = MAX_EMBED_DESCRIPTION - current_message_length + + if len(alert_content) > remaining_chars: + url = await upload_log([ctx.message], bot.instance.user.id, ctx.attachments) + log_site_msg = f"The full message can be found [here]({url})" + # 7 because that's the length of "[...]\n\n" + alert_content = alert_content[:remaining_chars - (7 + len(log_site_msg))] + "[...]\n\n" + log_site_msg + + return alert_content + + +async def build_mod_alert(ctx: FilterContext, triggered_filters: dict[FilterList, list[str]]) -> Embed: + """Build an alert message from the filter context.""" + embed = Embed(color=Colours.soft_orange) + embed.set_thumbnail(url=ctx.author.display_avatar.url) + triggered_by = f"**Triggered by:** {format_user(ctx.author)}" + if ctx.channel.guild: + triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}\n" + else: + triggered_in = "**Triggered in:** :warning:**DM**:warning:\n" + if len(ctx.related_channels) > 1: + triggered_in += f"**Channels:** {', '.join(channel.mention for channel in ctx.related_channels)}\n" + + filters = [] + for filter_list, list_message in triggered_filters.items(): + if list_message: + filters.append(f"**{filter_list.name.title()} Filters:** {', '.join(list_message)}") + filters = "\n".join(filters) + + matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) if ctx.matches else "" + actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") + + mod_alert_message = "\n".join(part for part in (triggered_by, triggered_in, filters, matches, actions) if part) + mod_alert_message += f"\n**[Original Content]({ctx.message.jump_url})**:\n" + mod_alert_message += await _build_alert_message_content(ctx, len(mod_alert_message)) + + embed.description = mod_alert_message + return embed + + def populate_embed_from_dict(embed: Embed, data: dict) -> None: """Populate a Discord embed by populating fields from the given dict.""" for setting, value in data.items(): diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index d5dfbfc830..86b6ab1011 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -31,7 +31,7 @@ def subclasses_in_package(package: str, prefix: str, parent: T) -> set[T]: # Find all classes in each module... for _, class_ in inspect.getmembers(module, inspect.isclass): # That are a subclass of the given class. - if parent in class_.__bases__: + if parent in class_.__mro__: subclasses.add(class_) return subclasses diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 44eaa5ea72..514ef39e13 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -8,16 +8,15 @@ import discord from botcore.site_api import ResponseCodeError -from discord import Colour, Embed, HTTPException, Message +from discord import Colour, Embed, HTTPException, Message, MessageType from discord.ext import commands from discord.ext.commands import BadArgument, Cog, Context, has_any_role -from discord.utils import escape_markdown import bot import bot.exts.filtering._ui.filter as filters_ui from bot import constants from bot.bot import Bot -from bot.constants import Channels, Colours, MODERATION_ROLES, Roles, Webhooks +from bot.constants import Channels, MODERATION_ROLES, Roles, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filter_lists.filter_list import AtomicList @@ -28,14 +27,18 @@ ) from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter from bot.exts.filtering._ui.search import SearchEditView, search_criteria_converter -from bot.exts.filtering._ui.ui import ArgumentCompletionView, DeleteConfirmationView, format_response_error +from bot.exts.filtering._ui.ui import ( + ArgumentCompletionView, DeleteConfirmationView, build_mod_alert, format_response_error +) from bot.exts.filtering._utils import past_tense, repr_equals, starting_value, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator -from bot.utils.messages import format_channel, format_user +from bot.utils.message_cache import MessageCache log = get_logger(__name__) +CACHE_SIZE = 100 + class Filtering(Cog): """Filtering and alerting for content posted on the server.""" @@ -55,6 +58,8 @@ def __init__(self, bot: Bot): self.loaded_filters = {} self.loaded_filter_settings = {} + self.message_cache = MessageCache(CACHE_SIZE, newest_first=True) + async def cog_load(self) -> None: """ Fetch the filter data from the API, parse it, and load it to the appropriate data structures. @@ -165,8 +170,9 @@ async def cog_check(self, ctx: Context) -> bool: @Cog.listener() async def on_message(self, msg: Message) -> None: """Filter the contents of a sent message.""" - if msg.author.bot or msg.webhook_id: + if msg.author.bot or msg.webhook_id or msg.type == MessageType.auto_moderation_action: return + self.message_cache.append(msg) ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) @@ -766,32 +772,8 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi return name = f"{ctx.event.name.replace('_', ' ').title()} Filter" - - embed = Embed(color=Colours.soft_orange) - embed.set_thumbnail(url=ctx.author.display_avatar.url) - triggered_by = f"**Triggered by:** {format_user(ctx.author)}" - if ctx.channel.guild: - triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}\n" - else: - triggered_in = "**Triggered in:** :warning:**DM**:warning:\n" - - filters = [] - for filter_list, list_message in triggered_filters.items(): - if list_message: - filters.append(f"**{filter_list.name.title()} Filters:** {', '.join(list_message)}") - filters = "\n".join(filters) - - matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) - actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") - content = f"**[Original Content]({ctx.message.jump_url})**:\n{escape_markdown(ctx.content)}" - - embed_content = "\n".join( - part for part in (triggered_by, triggered_in, filters, matches, actions, content) if part - ) - if len(embed_content) > 4000: - embed_content = embed_content[:4000] + " [...]" - embed.description = embed_content - + embed = await build_mod_alert(ctx, triggered_filters) + # There shouldn't be more than 10, but if there are it's not very useful to send them all. await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10]) async def _resolve_list_type_and_name( diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index fd9404b1ad..aee751345c 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -19,6 +19,7 @@ from bot.exts.moderation.modlog import ModLog from bot.log import get_logger from bot.utils.channel import is_mod_channel +from bot.utils.messages import upload_log log = get_logger(__name__) @@ -351,7 +352,7 @@ async def _modlog_cleaned_messages( # Reverse the list to have reverse chronological order log_messages = reversed(messages) - log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) + log_url = await upload_log(log_messages, ctx.author.id) # Build the embed and send it if channels == "*": diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index efa87ce251..511f05c509 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -3,23 +3,20 @@ import itertools import typing as t from datetime import datetime, timezone -from itertools import zip_longest import discord -from botcore.site_api import ResponseCodeError from dateutil.relativedelta import relativedelta from deepdiff import DeepDiff from discord import Colour, Message, Thread from discord.abc import GuildChannel from discord.ext.commands import Cog, Context from discord.utils import escape_markdown, format_dt, snowflake_time -from sentry_sdk import add_breadcrumb from bot.bot import Bot -from bot.constants import Categories, Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, Roles, URLs +from bot.constants import Categories, Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, Roles from bot.log import get_logger from bot.utils import time -from bot.utils.messages import format_user +from bot.utils.messages import format_user, upload_log log = get_logger(__name__) @@ -45,48 +42,6 @@ def __init__(self, bot: Bot): self._cached_edits = [] - async def upload_log( - self, - messages: t.Iterable[discord.Message], - actor_id: int, - attachments: t.Iterable[t.List[str]] = None - ) -> str: - """Upload message logs to the database and return a URL to a page for viewing the logs.""" - if attachments is None: - attachments = [] - - deletedmessage_set = [ - { - "id": message.id, - "author": message.author.id, - "channel_id": message.channel.id, - "content": message.content.replace("\0", ""), # Null chars cause 400. - "embeds": [embed.to_dict() for embed in message.embeds], - "attachments": attachment, - } - for message, attachment in zip_longest(messages, attachments, fillvalue=[]) - ] - - try: - response = await self.bot.api_client.post( - "bot/deleted-messages", - json={ - "actor": actor_id, - "creation": datetime.now(timezone.utc).isoformat(), - "deletedmessage_set": deletedmessage_set, - } - ) - except ResponseCodeError as e: - add_breadcrumb( - category="api_error", - message=str(e), - level="error", - data=deletedmessage_set, - ) - raise - - return f"{URLs.site_logs_view}/{response['id']}" - def ignore(self, event: Event, *items: int) -> None: """Add event to ignored events to suppress log emission.""" for item in items: @@ -609,7 +564,7 @@ async def log_cached_deleted_message(self, message: discord.Message) -> None: remaining_chars = 4090 - len(response) if len(content) > remaining_chars: - botlog_url = await self.upload_log(messages=[message], actor_id=message.author.id) + botlog_url = await upload_log(messages=[message], actor_id=message.author.id) ending = f"\n\nMessage truncated, [full message here]({botlog_url})." truncation_point = remaining_chars - len(ending) content = f"{content[:truncation_point]}...{ending}" diff --git a/bot/utils/messages.py b/bot/utils/messages.py index 63929cd0b8..c5f6dc41aa 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,16 +1,21 @@ import asyncio import random import re +from collections.abc import Iterable +from datetime import datetime, timezone from functools import partial from io import BytesIO from typing import Callable, List, Optional, Sequence, Union import discord +from botcore.site_api import ResponseCodeError from botcore.utils import scheduling +from discord import Message from discord.ext.commands import Context +from sentry_sdk import add_breadcrumb import bot -from bot.constants import Emojis, MODERATION_ROLES, NEGATIVE_REPLIES +from bot.constants import Emojis, MODERATION_ROLES, NEGATIVE_REPLIES, URLs from bot.log import get_logger log = get_logger(__name__) @@ -235,7 +240,7 @@ async def send_denial(ctx: Context, reason: str) -> discord.Message: return await ctx.send(embed=embed) -def format_user(user: discord.abc.User) -> str: +def format_user(user: discord.User | discord.Member) -> str: """Return a string for `user` which has their mention and ID.""" return f"{user.mention} (`{user.id}`)" @@ -247,3 +252,43 @@ def format_channel(channel: discord.abc.Messageable) -> str: formatted += f"/{channel.parent}" formatted += ")" return formatted + + +async def upload_log(messages: Iterable[Message], actor_id: int, attachments: dict[int, list[str]] = None) -> str: + """Upload message logs to the database and return a URL to a page for viewing the logs.""" + if attachments is None: + attachments = [] + else: + attachments = [attachments.get(message.id, []) for message in messages] + + deletedmessage_set = [ + { + "id": message.id, + "author": message.author.id, + "channel_id": message.channel.id, + "content": message.content.replace("\0", ""), # Null chars cause 400. + "embeds": [embed.to_dict() for embed in message.embeds], + "attachments": attachment, + } + for message, attachment in zip(messages, attachments) + ] + + try: + response = await bot.instance.api_client.post( + "bot/deleted-messages", + json={ + "actor": actor_id, + "creation": datetime.now(timezone.utc).isoformat(), + "deletedmessage_set": deletedmessage_set, + } + ) + except ResponseCodeError as e: + add_breadcrumb( + category="api_error", + message=str(e), + level="error", + data=deletedmessage_set, + ) + raise + + return f"{URLs.site_logs_view}/{response['id']}" From c20398233a4a792e3207d52765aaf530a468351a Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 1 Nov 2022 20:57:09 +0200 Subject: [PATCH 062/132] Add the rest of the antispam rules This is mostly a copy-paste of the implementations in the old system into the new system's structure. The mentions rule required changing the `triggers_on` method to async. --- bot/exts/filtering/_filter_lists/antispam.py | 10 ++- bot/exts/filtering/_filter_lists/domain.py | 2 +- bot/exts/filtering/_filter_lists/extension.py | 4 +- .../filtering/_filter_lists/filter_list.py | 16 ++-- bot/exts/filtering/_filter_lists/invite.py | 5 +- bot/exts/filtering/_filter_lists/token.py | 2 +- bot/exts/filtering/_filter_lists/unique.py | 2 +- .../_filters/antispam/attachments.py | 43 +++++++++ bot/exts/filtering/_filters/antispam/burst.py | 41 +++++++++ bot/exts/filtering/_filters/antispam/chars.py | 43 +++++++++ .../filtering/_filters/antispam/duplicates.py | 4 +- bot/exts/filtering/_filters/antispam/emoji.py | 53 +++++++++++ bot/exts/filtering/_filters/antispam/links.py | 52 +++++++++++ .../filtering/_filters/antispam/mentions.py | 90 +++++++++++++++++++ .../filtering/_filters/antispam/newlines.py | 61 +++++++++++++ .../_filters/antispam/role_mentions.py | 42 +++++++++ bot/exts/filtering/_filters/domain.py | 2 +- bot/exts/filtering/_filters/extension.py | 2 +- bot/exts/filtering/_filters/filter.py | 6 +- bot/exts/filtering/_filters/invite.py | 2 +- bot/exts/filtering/_filters/token.py | 2 +- .../_filters/unique/discord_token.py | 2 +- .../filtering/_filters/unique/everyone.py | 2 +- .../filtering/_filters/unique/rich_embed.py | 2 +- bot/exts/filtering/_filters/unique/webhook.py | 2 +- bot/exts/filtering/filtering.py | 5 +- tests/bot/exts/filtering/test_filters.py | 6 +- 27 files changed, 468 insertions(+), 35 deletions(-) create mode 100644 bot/exts/filtering/_filters/antispam/attachments.py create mode 100644 bot/exts/filtering/_filters/antispam/burst.py create mode 100644 bot/exts/filtering/_filters/antispam/chars.py create mode 100644 bot/exts/filtering/_filters/antispam/emoji.py create mode 100644 bot/exts/filtering/_filters/antispam/links.py create mode 100644 bot/exts/filtering/_filters/antispam/mentions.py create mode 100644 bot/exts/filtering/_filters/antispam/newlines.py create mode 100644 bot/exts/filtering/_filters/antispam/role_mentions.py diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index 2dab54ce62..b2f8730948 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -34,7 +34,9 @@ class AntispamList(UniquesListBase): """ A list of anti-spam rules. - Messages from the last X seconds is passed to each rule, which decide whether it triggers across those messages. + Messages from the last X seconds are passed to each rule, which decides whether it triggers across those messages. + + The infraction reason is set dynamically. """ name = "antispam" @@ -67,7 +69,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.filtering_cog.message_cache) ) new_ctx = ctx.replace(content=relevant_messages) - triggers = sublist.filter_list_result(new_ctx) + triggers = await sublist.filter_list_result(new_ctx) if not triggers: return None, [] @@ -88,7 +90,9 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, # Smaller infraction value = higher in hierarchy. if not current_infraction or new_infraction.infraction_type.value < current_infraction.value: # Pick the first triggered filter for the reason, there's no good way to decide between them. - new_infraction.infraction_reason = f"{triggers[0].name} spam - {ctx.filter_info[triggers[0]]}" + new_infraction.infraction_reason = ( + f"{triggers[0].name.replace('_', ' ')} spam – {ctx.filter_info[triggers[0]]}" + ) current_actions["infraction_and_notification"] = new_infraction self.message_deletion_queue[ctx.author].current_infraction = new_infraction.infraction_type else: diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index d97aa252ba..0b56e8d730 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -52,7 +52,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, urls = {match.group(1).lower().rstrip("/") for match in URL_RE.finditer(text)} new_ctx = ctx.replace(content=urls) - triggers = self[ListType.DENY].filter_list_result(new_ctx) + triggers = await self[ListType.DENY].filter_list_result(new_ctx) ctx.notification_domain = new_ctx.notification_domain actions = None messages = [] diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index 3f9d2b2870..a53520bf71 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -76,7 +76,9 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, (splitext(attachment.filename.lower())[1], attachment.filename) for attachment in ctx.message.attachments } new_ctx = ctx.replace(content={ext for ext, _ in all_ext}) # And prepare the context for the filters to read. - triggered = [filter_ for filter_ in self[ListType.ALLOW].filters.values() if filter_.triggered_on(new_ctx)] + triggered = [ + filter_ for filter_ in self[ListType.ALLOW].filters.values() if await filter_.triggered_on(new_ctx) + ] allowed_ext = {filter_.content for filter_ in triggered} # Get the extensions in the message that are allowed. # See if there are any extensions left which aren't allowed. diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index f9db54a21f..938766acae 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -65,7 +65,7 @@ def label(self) -> str: """Provide a short description identifying the list with its name and type.""" return f"{past_tense(self.list_type.name.lower())} {self.name.lower()}" - def filter_list_result(self, ctx: FilterContext) -> list[Filter]: + async def filter_list_result(self, ctx: FilterContext) -> list[Filter]: """ Sift through the list of filters, and return only the ones which apply to the given context. @@ -79,10 +79,12 @@ def filter_list_result(self, ctx: FilterContext) -> list[Filter]: If the filter is relevant in context, see if it actually triggers. """ - return self._create_filter_list_result(ctx, self.defaults, self.filters.values()) + return await self._create_filter_list_result(ctx, self.defaults, self.filters.values()) @staticmethod - def _create_filter_list_result(ctx: FilterContext, defaults: Defaults, filters: Iterable[Filter]) -> list[Filter]: + async def _create_filter_list_result( + ctx: FilterContext, defaults: Defaults, filters: Iterable[Filter] + ) -> list[Filter]: """A helper function to evaluate the result of `filter_list_result`.""" passed_by_default, failed_by_default = defaults.validations.evaluate(ctx) default_answer = not bool(failed_by_default) @@ -90,12 +92,12 @@ def _create_filter_list_result(ctx: FilterContext, defaults: Defaults, filters: relevant_filters = [] for filter_ in filters: if not filter_.validations: - if default_answer and filter_.triggered_on(ctx): + if default_answer and await filter_.triggered_on(ctx): relevant_filters.append(filter_) else: passed, failed = filter_.validations.evaluate(ctx) if not failed and failed_by_default < passed: - if filter_.triggered_on(ctx): + if await filter_.triggered_on(ctx): relevant_filters.append(filter_) return relevant_filters @@ -222,10 +224,10 @@ def subscribe(self, filter_: UniqueFilter, *events: Event) -> None: if filter_ not in self.subscriptions[event]: self.subscriptions[event].append(filter_.id) - def filter_list_result(self, ctx: FilterContext) -> list[Filter]: + async def filter_list_result(self, ctx: FilterContext) -> list[Filter]: """Sift through the list of filters, and return only the ones which apply to the given context.""" event_filters = [self.filters[id_] for id_ in self.subscriptions[ctx.event]] - return self._create_filter_list_result(ctx, self.defaults, event_filters) + return await self._create_filter_list_result(ctx, self.defaults, event_filters) class UniquesListBase(FilterList[UniqueFilter]): diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 0b84aec0e7..911b951dd1 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -81,7 +81,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, # Find any blocked invites new_ctx = ctx.replace(content={invite.guild.id for invite in invites_for_inspection.values()}) - triggered = self[ListType.DENY].filter_list_result(new_ctx) + triggered = await self[ListType.DENY].filter_list_result(new_ctx) blocked_guilds = {filter_.content for filter_ in triggered} blocked_invites = { code: invite for code, invite in invites_for_inspection.items() if invite.guild.id in blocked_guilds @@ -100,7 +100,8 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, if check_if_allowed: # Whether unknown invites need to be checked. new_ctx = ctx.replace(content=guilds_for_inspection) allowed = { - filter_.content for filter_ in self[ListType.ALLOW].filters.values() if filter_.triggered_on(new_ctx) + filter_.content for filter_ in self[ListType.ALLOW].filters.values() + if await filter_.triggered_on(new_ctx) } unknown_invites.update({ code: invite for code, invite in invites_for_inspection.items() if invite.guild.id not in allowed diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index c7d7cb4447..274dc5ea75 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -53,7 +53,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, text = clean_input(text) ctx = ctx.replace(content=text) - triggers = self[ListType.DENY].filter_list_result(ctx) + triggers = await self[ListType.DENY].filter_list_result(ctx) actions = None messages = [] if triggers: diff --git a/bot/exts/filtering/_filter_lists/unique.py b/bot/exts/filtering/_filter_lists/unique.py index 5204065f94..ecc49af870 100644 --- a/bot/exts/filtering/_filter_lists/unique.py +++ b/bot/exts/filtering/_filter_lists/unique.py @@ -31,7 +31,7 @@ def get_filter_type(self, content: str) -> type[UniqueFilter] | None: async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" - triggers = self[ListType.DENY].filter_list_result(ctx) + triggers = await self[ListType.DENY].filter_list_result(ctx) actions = None messages = [] if triggers: diff --git a/bot/exts/filtering/_filters/antispam/attachments.py b/bot/exts/filtering/_filters/antispam/attachments.py new file mode 100644 index 0000000000..216d9b886e --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/attachments.py @@ -0,0 +1,43 @@ +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + + +class ExtraAttachmentsSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of attachments before the filter is triggered." + + interval: int = 10 + threshold: int = 6 + + +class AttachmentsFilter(UniqueFilter): + """Detects too many attachments sent by a single user.""" + + name = "attachments" + events = (Event.MESSAGE,) + extra_fields_type = ExtraAttachmentsSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author and len(msg.attachments) > 0} + total_recent_attachments = sum(len(msg.attachments) for msg in detected_messages) + + if total_recent_attachments > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {total_recent_attachments} attachments" + return True + return False diff --git a/bot/exts/filtering/_filters/antispam/burst.py b/bot/exts/filtering/_filters/antispam/burst.py new file mode 100644 index 0000000000..d78107d0a4 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/burst.py @@ -0,0 +1,41 @@ +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + + +class ExtraBurstSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of messages before the filter is triggered." + + interval: int = 10 + threshold: int = 7 + + +class BurstFilter(UniqueFilter): + """Detects too many messages sent by a single user.""" + + name = "burst" + events = (Event.MESSAGE,) + extra_fields_type = ExtraBurstSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author} + if len(detected_messages) > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {len(detected_messages)} messages" + return True + return False diff --git a/bot/exts/filtering/_filters/antispam/chars.py b/bot/exts/filtering/_filters/antispam/chars.py new file mode 100644 index 0000000000..5c4fa201cb --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/chars.py @@ -0,0 +1,43 @@ +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + + +class ExtraCharsSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of characters before the filter is triggered." + + interval: int = 5 + threshold: int = 4_200 + + +class CharsFilter(UniqueFilter): + """Detects too many characters sent by a single user.""" + + name = "chars" + events = (Event.MESSAGE,) + extra_fields_type = ExtraCharsSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author} + total_recent_chars = sum(len(msg.content) for msg in relevant_messages) + + if total_recent_chars > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {total_recent_chars} characters" + return True + return False diff --git a/bot/exts/filtering/_filters/antispam/duplicates.py b/bot/exts/filtering/_filters/antispam/duplicates.py index 5df2bb5c0a..60d5c322cf 100644 --- a/bot/exts/filtering/_filters/antispam/duplicates.py +++ b/bot/exts/filtering/_filters/antispam/duplicates.py @@ -15,7 +15,7 @@ class ExtraDuplicatesSettings(BaseModel): interval_description: ClassVar[str] = ( "Look for rule violations in messages from the last `interval` number of seconds." ) - threshold_description: ClassVar[str] = "Number of duplicate messages required to trigger the filter." + threshold_description: ClassVar[str] = "Maximum number of duplicate messages before the filter is triggered." interval: int = 10 threshold: int = 3 @@ -28,7 +28,7 @@ class DuplicatesFilter(UniqueFilter): events = (Event.MESSAGE,) extra_fields_type = ExtraDuplicatesSettings - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Search for the filter's content within a given context.""" earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) diff --git a/bot/exts/filtering/_filters/antispam/emoji.py b/bot/exts/filtering/_filters/antispam/emoji.py new file mode 100644 index 0000000000..0511e4a7b5 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/emoji.py @@ -0,0 +1,53 @@ +import re +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from emoji import demojize +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + +DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>|:\w+:") +CODE_BLOCK_RE = re.compile(r"```.*?```", flags=re.DOTALL) + + +class ExtraEmojiSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of emojis before the filter is triggered." + + interval: int = 10 + threshold: int = 20 + + +class EmojiFilter(UniqueFilter): + """Detects too many emojis sent by a single user.""" + + name = "emoji" + events = (Event.MESSAGE,) + extra_fields_type = ExtraEmojiSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author} + + # Get rid of code blocks in the message before searching for emojis. + # Convert Unicode emojis to :emoji: format to get their count. + total_emojis = sum( + len(DISCORD_EMOJI_RE.findall(demojize(CODE_BLOCK_RE.sub("", msg.content)))) + for msg in relevant_messages + ) + + if total_emojis > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {total_emojis} emojis" + return True + return False diff --git a/bot/exts/filtering/_filters/antispam/links.py b/bot/exts/filtering/_filters/antispam/links.py new file mode 100644 index 0000000000..76fe53e702 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/links.py @@ -0,0 +1,52 @@ +import re +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + +LINK_RE = re.compile(r"(https?://\S+)") + + +class ExtraLinksSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of links before the filter is triggered." + + interval: int = 10 + threshold: int = 10 + + +class DuplicatesFilter(UniqueFilter): + """Detects too many links sent by a single user.""" + + name = "links" + events = (Event.MESSAGE,) + extra_fields_type = ExtraLinksSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author} + + total_links = 0 + messages_with_links = 0 + for msg in relevant_messages: + total_matches = len(LINK_RE.findall(msg.content)) + if total_matches: + messages_with_links += 1 + total_links += total_matches + + if total_links > self.extra_fields.threshold and messages_with_links > 1: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {total_links} links" + return True + return False diff --git a/bot/exts/filtering/_filters/antispam/mentions.py b/bot/exts/filtering/_filters/antispam/mentions.py new file mode 100644 index 0000000000..29a2d56060 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/mentions.py @@ -0,0 +1,90 @@ +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from botcore.utils.logging import get_logger +from discord import DeletedReferencedMessage, MessageType, NotFound +from pydantic import BaseModel + +import bot +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + +log = get_logger(__name__) + + +class ExtraMentionsSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of distinct mentions before the filter is triggered." + + interval: int = 10 + threshold: int = 5 + + +class DuplicatesFilter(UniqueFilter): + """ + Detects total mentions exceeding the limit sent by a single user. + + Excludes mentions that are bots, themselves, or replied users. + + In very rare cases, may not be able to determine a + mention was to a reply, in which case it is not ignored. + """ + + name = "mentions" + events = (Event.MESSAGE,) + extra_fields_type = ExtraMentionsSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author} + + # We use `msg.mentions` here as that is supplied by the api itself, to determine who was mentioned. + # Additionally, `msg.mentions` includes the user replied to, even if the mention doesn't occur in the body. + # In order to exclude users who are mentioned as a reply, we check if the msg has a reference + # + # While we could use regex to parse the message content, and get a list of + # the mentions, that solution is very prone to breaking. + # We would need to deal with codeblocks, escaping markdown, and any discrepancies between + # our implementation and discord's Markdown parser which would cause false positives or false negatives. + total_recent_mentions = 0 + for msg in relevant_messages: + # We check if the message is a reply, and if it is try to get the author + # since we ignore mentions of a user that we're replying to + reply_author = None + + if msg.type == MessageType.reply: + ref = msg.reference + + if not (resolved := ref.resolved): + # It is possible, in a very unusual situation, for a message to have a reference + # that is both not in the cache, and deleted while running this function. + # In such a situation, this will throw an error which we catch. + try: + resolved = await bot.instance.get_partial_messageable(resolved.channel_id).fetch_message( + resolved.message_id + ) + except NotFound: + log.info('Could not fetch the reference message as it has been deleted.') + + if resolved and not isinstance(resolved, DeletedReferencedMessage): + reply_author = resolved.author + + for user in msg.mentions: + # Don't count bot or self mentions, or the user being replied to (if applicable) + if user.bot or user in {msg.author, reply_author}: + continue + total_recent_mentions += 1 + + if total_recent_mentions > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {total_recent_mentions} mentions" + return True + return False diff --git a/bot/exts/filtering/_filters/antispam/newlines.py b/bot/exts/filtering/_filters/antispam/newlines.py new file mode 100644 index 0000000000..b15a352197 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/newlines.py @@ -0,0 +1,61 @@ +import re +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + +NEWLINES = re.compile(r"(\n+)") + + +class ExtraNewlinesSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of newlines before the filter is triggered." + consecutive_threshold_description: ClassVar[str] = ( + "Maximum number of consecutive newlines before the filter is triggered." + ) + + interval: int = 10 + threshold: int = 100 + consecutive_threshold: int = 10 + + +class NewlinesFilter(UniqueFilter): + """Detects too many newlines sent by a single user.""" + + name = "newlines" + events = (Event.MESSAGE,) + extra_fields_type = ExtraNewlinesSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author} + + # Identify groups of newline characters and get group & total counts + newline_counts = [] + for msg in relevant_messages: + newline_counts += [len(group) for group in NEWLINES.findall(msg.content)] + total_recent_newlines = sum(newline_counts) + # Get maximum newline group size + max_newline_group = max(newline_counts, default=0) + + # Check first for total newlines, if this passes then check for large groupings + if total_recent_newlines > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {total_recent_newlines} newlines" + return True + if max_newline_group > self.extra_fields.consecutive_threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {max_newline_group} consecutive newlines" + return True + return False diff --git a/bot/exts/filtering/_filters/antispam/role_mentions.py b/bot/exts/filtering/_filters/antispam/role_mentions.py new file mode 100644 index 0000000000..49de642faa --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/role_mentions.py @@ -0,0 +1,42 @@ +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + + +class ExtraRoleMentionsSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Maximum number of role mentions before the filter is triggered." + + interval: int = 10 + threshold: int = 3 + + +class DuplicatesFilter(UniqueFilter): + """Detects too many role mentions sent by a single user.""" + + name = "role_mentions" + events = (Event.MESSAGE,) + extra_fields_type = ExtraRoleMentionsSettings + + async def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + detected_messages = {msg for msg in relevant_messages if msg.author == ctx.author} + total_recent_mentions = sum(len(msg.role_mentions) for msg in relevant_messages) + + if total_recent_mentions > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {total_recent_mentions} role mentions" + return True + return False diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index e22cafbb76..4cc3a6f5ae 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -34,7 +34,7 @@ class DomainFilter(Filter): name = "domain" extra_fields_type = ExtraDomainSettings - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Searches for a domain within a given context.""" domain = tldextract.extract(self.content).registered_domain diff --git a/bot/exts/filtering/_filters/extension.py b/bot/exts/filtering/_filters/extension.py index 926a6a2fb9..f3f64532f7 100644 --- a/bot/exts/filtering/_filters/extension.py +++ b/bot/exts/filtering/_filters/extension.py @@ -11,7 +11,7 @@ class ExtensionFilter(Filter): name = "extension" - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Searches for an attachment extension in the context content, given as a set of extensions.""" return self.content in ctx.content diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index b0d19d3a84..4ae7ec45f1 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Any from pydantic import ValidationError @@ -48,7 +48,7 @@ def overrides(self) -> tuple[dict[str, Any], dict[str, Any]]: return settings, filter_settings @abstractmethod - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Search for the filter's content within a given context.""" @classmethod @@ -81,7 +81,7 @@ def __str__(self) -> str: return string -class UniqueFilter(Filter, ABC): +class UniqueFilter(Filter): """ Unique filters are ones that should only be run once in a given context. diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py index ac4f62cb6d..e8f3e98519 100644 --- a/bot/exts/filtering/_filters/invite.py +++ b/bot/exts/filtering/_filters/invite.py @@ -20,7 +20,7 @@ def __init__(self, filter_data: dict, defaults_data: dict | None = None): super().__init__(filter_data, defaults_data) self.content = int(self.content) - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Searches for a guild ID in the context content, given as a set of IDs.""" return self.content in ctx.content diff --git a/bot/exts/filtering/_filters/token.py b/bot/exts/filtering/_filters/token.py index 04e30cb039..f61d388463 100644 --- a/bot/exts/filtering/_filters/token.py +++ b/bot/exts/filtering/_filters/token.py @@ -11,7 +11,7 @@ class TokenFilter(Filter): name = "token" - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Searches for a regex pattern within a given context.""" pattern = self.content diff --git a/bot/exts/filtering/_filters/unique/discord_token.py b/bot/exts/filtering/_filters/unique/discord_token.py index 7fdb800df6..731df198c8 100644 --- a/bot/exts/filtering/_filters/unique/discord_token.py +++ b/bot/exts/filtering/_filters/unique/discord_token.py @@ -69,7 +69,7 @@ def mod_log(self) -> ModLog | None: """Get currently loaded ModLog cog instance.""" return bot.instance.get_cog("ModLog") - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Return whether the message contains Discord client tokens.""" found_token = self.find_token_in_message(ctx.content) if not found_token: diff --git a/bot/exts/filtering/_filters/unique/everyone.py b/bot/exts/filtering/_filters/unique/everyone.py index 06d3a19bb3..a32e67cc57 100644 --- a/bot/exts/filtering/_filters/unique/everyone.py +++ b/bot/exts/filtering/_filters/unique/everyone.py @@ -18,7 +18,7 @@ class EveryoneFilter(UniqueFilter): name = "everyone" events = (Event.MESSAGE, Event.MESSAGE_EDIT) - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Search for the filter's content within a given context.""" # First pass to avoid running re.sub on every message if not EVERYONE_PING_RE.search(ctx.content): diff --git a/bot/exts/filtering/_filters/unique/rich_embed.py b/bot/exts/filtering/_filters/unique/rich_embed.py index a0d9e263f0..09d5133739 100644 --- a/bot/exts/filtering/_filters/unique/rich_embed.py +++ b/bot/exts/filtering/_filters/unique/rich_embed.py @@ -17,7 +17,7 @@ class RichEmbedFilter(UniqueFilter): name = "rich_embed" events = (Event.MESSAGE, Event.MESSAGE_EDIT) - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Determine if `msg` contains any rich embeds not auto-generated from a URL.""" if ctx.embeds: for embed in ctx.embeds: diff --git a/bot/exts/filtering/_filters/unique/webhook.py b/bot/exts/filtering/_filters/unique/webhook.py index b9d98db350..16ff1b213b 100644 --- a/bot/exts/filtering/_filters/unique/webhook.py +++ b/bot/exts/filtering/_filters/unique/webhook.py @@ -29,7 +29,7 @@ def mod_log(self) -> ModLog | None: """Get current instance of `ModLog`.""" return bot.instance.get_cog("ModLog") - def triggered_on(self, ctx: FilterContext) -> bool: + async def triggered_on(self, ctx: FilterContext) -> bool: """Search for a webhook in the given content. If found, attempt to delete it.""" matches = set(WEBHOOK_URL_RE.finditer(ctx.content)) if not matches: diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 514ef39e13..aad36af145 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -175,7 +175,6 @@ async def on_message(self, msg: Message) -> None: self.message_cache.append(msg) ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) - result_actions, list_messages = await self._resolve_action(ctx) if result_actions: await result_actions.action(ctx) @@ -194,7 +193,7 @@ async def blocklist(self, ctx: Context) -> None: @blocklist.command(name="list", aliases=("get",)) async def bl_list(self, ctx: Context, list_name: Optional[str] = None) -> None: """List the contents of a specified blacklist.""" - result = self._resolve_list_type_and_name(ctx, ListType.DENY, list_name) + result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name) if not result: return list_type, filter_list = result @@ -237,7 +236,7 @@ async def allowlist(self, ctx: Context) -> None: @allowlist.command(name="list", aliases=("get",)) async def al_list(self, ctx: Context, list_name: Optional[str] = None) -> None: """List the contents of a specified whitelist.""" - result = self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name) + result = await self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name) if not result: return list_type, filter_list = result diff --git a/tests/bot/exts/filtering/test_filters.py b/tests/bot/exts/filtering/test_filters.py index 214637b525..29b50188a2 100644 --- a/tests/bot/exts/filtering/test_filters.py +++ b/tests/bot/exts/filtering/test_filters.py @@ -5,7 +5,7 @@ from tests.helpers import MockMember, MockMessage, MockTextChannel -class FilterTests(unittest.TestCase): +class FilterTests(unittest.IsolatedAsyncioTestCase): """Test functionality of the token filter.""" def setUp(self) -> None: @@ -14,7 +14,7 @@ def setUp(self) -> None: message = MockMessage(author=member, channel=channel) self.ctx = FilterContext(Event.MESSAGE, member, channel, "", message) - def test_token_filter_triggers(self): + async def test_token_filter_triggers(self): """The filter should evaluate to True only if its token is found in the context content.""" test_cases = ( (r"hi", "oh hi there", True), @@ -37,5 +37,5 @@ def test_token_filter_triggers(self): "additional_field": "{}" # noqa: P103 }) self.ctx.content = content - result = filter_.triggered_on(self.ctx) + result = await filter_.triggered_on(self.ctx) self.assertEqual(result, expected) From e5be580371045cc1650c73d1f8718d493ce8f457 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 1 Nov 2022 21:49:45 +0200 Subject: [PATCH 063/132] Fix argument completion for al and bl --- bot/exts/filtering/filtering.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index aad36af145..e366ea08d2 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -193,7 +193,7 @@ async def blocklist(self, ctx: Context) -> None: @blocklist.command(name="list", aliases=("get",)) async def bl_list(self, ctx: Context, list_name: Optional[str] = None) -> None: """List the contents of a specified blacklist.""" - result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name) + result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name, exclude="list_type") if not result: return list_type, filter_list = result @@ -218,7 +218,7 @@ async def bl_add( The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. """ - result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name) + result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name, exclude="list_type") if result is None: return list_type, filter_list = result @@ -236,7 +236,7 @@ async def allowlist(self, ctx: Context) -> None: @allowlist.command(name="list", aliases=("get",)) async def al_list(self, ctx: Context, list_name: Optional[str] = None) -> None: """List the contents of a specified whitelist.""" - result = await self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name) + result = await self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name, exclude="list_type") if not result: return list_type, filter_list = result @@ -261,7 +261,7 @@ async def al_add( The settings can be provided in the command itself, in the format of `setting_name=value` (no spaces around the equal sign). The value doesn't need to (shouldn't) be surrounded in quotes even if it contains spaces. """ - result = await self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name) + result = await self._resolve_list_type_and_name(ctx, ListType.ALLOW, list_name, exclude="list_type") if result is None: return list_type, filter_list = result @@ -776,23 +776,25 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10]) async def _resolve_list_type_and_name( - self, ctx: Context, list_type: Optional[ListType] = None, list_name: Optional[str] = None - ) -> Optional[tuple[ListType, FilterList]]: + self, ctx: Context, list_type: ListType | None = None, list_name: str | None = None, *, exclude: str = "" + ) -> tuple[ListType, FilterList] | None: """Prompt the user to complete the list type or list name if one of them is missing.""" if list_name is None: + args = [list_type] if exclude != "list_type" else [] await ctx.send( "The **list_name** argument is unspecified. Please pick a value from the options below:", - view=ArgumentCompletionView(ctx, [list_type], "list_name", list(self.filter_lists), 1, None) + view=ArgumentCompletionView(ctx, args, "list_name", list(self.filter_lists), 1, None) ) return None filter_list = self._get_list_by_name(list_name) if list_type is None: if len(filter_list) > 1: + args = [list_name] if exclude != "list_name" else [] await ctx.send( "The **list_type** argument is unspecified. Please pick a value from the options below:", view=ArgumentCompletionView( - ctx, [list_name], "list_type", [option.name for option in ListType], 0, list_type_converter + ctx, args, "list_type", [option.name for option in ListType], 0, list_type_converter ) ) return None From 6521ad32a7fa140e736d264ca2a79720661327e4 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 1 Nov 2022 22:19:10 +0200 Subject: [PATCH 064/132] Prettify f and fl describe output --- bot/exts/filtering/filtering.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index e366ea08d2..34ca9f6e5e 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -321,8 +321,10 @@ async def f_list( async def f_describe(self, ctx: Context, filter_name: Optional[str]) -> None: """Show a description of the specified filter, or a list of possible values if no name is specified.""" if not filter_name: - embed = Embed(description="\n".join(self.loaded_filters)) + filter_names = [f"» {f}" for f in self.loaded_filters] + embed = Embed(colour=Colour.blue()) embed.set_author(name="List of filter names") + await LinePaginator.paginate(filter_names, ctx, embed, max_lines=10, empty=False) else: filter_type = self.loaded_filters.get(filter_name) if not filter_type: @@ -331,10 +333,9 @@ async def f_describe(self, ctx: Context, filter_name: Optional[str]) -> None: await ctx.send(f":x: There's no filter type named {filter_name!r}.") return # Use the class's docstring, and ignore single newlines. - embed = Embed(description=re.sub(r"(? None: """Show a description of the specified filter list, or a list of possible values if no values are provided.""" if not list_type and not list_name: - embed = Embed(description="\n".join(f"\u2003 {fl}" for fl in self.filter_lists), colour=Colour.blue()) + list_names = [f"» {fl}" for fl in self.filter_lists] + embed = Embed(colour=Colour.blue()) embed.set_author(name="List of filter lists names") - await ctx.send(embed=embed) + await LinePaginator.paginate(list_names, ctx, embed, max_lines=10, empty=False) return result = await self._resolve_list_type_and_name(ctx, list_type, list_name) From 00c6dbc194f8e45060af235702356ea581b2bc2f Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 1 Nov 2022 22:52:46 +0200 Subject: [PATCH 065/132] Remove old filtering constants --- bot/constants.py | 36 ----------------- config-default.yml | 97 ---------------------------------------------- 2 files changed, 133 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 51104ecb7a..8a2571a985 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -213,30 +213,6 @@ class Redis(metaclass=YAMLGetter): use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis -class Filter(metaclass=YAMLGetter): - section = "filter" - - filter_domains: bool - filter_everyone_ping: bool - filter_invites: bool - filter_zalgo: bool - watch_regex: bool - watch_rich_embeds: bool - - # Notifications are not expected for "watchlist" type filters - - notify_user_domains: bool - notify_user_everyone_ping: bool - notify_user_invites: bool - notify_user_zalgo: bool - - offensive_msg_delete_days: int - ping_everyone: bool - - channel_whitelist: List[int] - role_whitelist: List[int] - - class Cooldowns(metaclass=YAMLGetter): section = "bot" subsection = "cooldowns" @@ -580,18 +556,6 @@ class Metabase(metaclass=YAMLGetter): max_session_age: int -class AntiSpam(metaclass=YAMLGetter): - section = 'anti_spam' - - cache_size: int - - clean_offending: bool - ping_everyone: bool - - punishment: Dict[str, Dict[str, int]] - rules: Dict[str, Dict[str, int]] - - class BigBrother(metaclass=YAMLGetter): section = 'big_brother' diff --git a/config-default.yml b/config-default.yml index c28e430704..4407177d97 100644 --- a/config-default.yml +++ b/config-default.yml @@ -327,45 +327,6 @@ guild: filters: 926442964463521843 -filter: - # What do we filter? - filter_domains: true - filter_everyone_ping: true - filter_invites: true - filter_zalgo: false - watch_regex: true - watch_rich_embeds: true - - # Notify user on filter? - # Notifications are not expected for "watchlist" type filters - notify_user_domains: false - notify_user_everyone_ping: true - notify_user_invites: true - notify_user_zalgo: false - - # Filter configuration - offensive_msg_delete_days: 7 # How many days before deleting an offensive message? - ping_everyone: true - - # Censor doesn't apply to these - channel_whitelist: - - *ADMINS - - *BB_LOGS - - *DEV_LOG - - *MESSAGE_LOG - - *MOD_LOG - - *STAFF_LOUNGE - - role_whitelist: - - *ADMINS_ROLE - - *HELPERS_ROLE - - *MODS_ROLE - - *OWNERS_ROLE - - *PY_COMMUNITY_ROLE - - *SPRINTERS - - *PY_PARTNER_ROLE - - keys: github: !ENV "GITHUB_API_KEY" site_api: !ENV "BOT_API_KEY" @@ -398,64 +359,6 @@ urls: github_bot_repo: "https://github.com/python-discord/bot" -anti_spam: - cache_size: 100 - - # Clean messages that violate a rule. - clean_offending: true - ping_everyone: true - - punishment: - remove_after: 600 - role_id: *MUTED_ROLE - - rules: - attachments: - interval: 10 - max: 6 - - burst: - interval: 10 - max: 7 - - # Burst shared it (temporarily) disabled to prevent - # the bug that triggers multiple infractions/DMs per - # user. It also tends to catch a lot of innocent users - # now that we're so big. - # burst_shared: - # interval: 10 - # max: 20 - - chars: - interval: 5 - max: 4_200 - - discord_emojis: - interval: 10 - max: 20 - - duplicates: - interval: 10 - max: 3 - - links: - interval: 10 - max: 10 - - mentions: - interval: 10 - max: 5 - - newlines: - interval: 10 - max: 100 - max_consecutive: 10 - - role_mentions: - interval: 10 - max: 3 - - metabase: username: !ENV "METABASE_USERNAME" password: !ENV "METABASE_PASSWORD" From e100ae9b63f8fbb075e7ab5793d5028c74c4607b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 4 Nov 2022 00:28:52 +0200 Subject: [PATCH 066/132] Stop using None as a valid setting value A None value signifies that there's no override, and it shouldn't be used for anything else. Doing so is confusing and bug-prone. --- .../actions/infraction_and_notification.py | 35 ++++++++++--------- bot/exts/filtering/_ui/ui.py | 30 ++++++---------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index fb679855a0..b8b4636268 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -61,6 +61,7 @@ class Infraction(Enum): WARNING = auto() WATCH = auto() NOTE = auto() + NONE = auto() def __str__(self) -> str: return self.name @@ -70,8 +71,8 @@ async def invoke( user: Member | User, channel: discord.abc.Messageable, alerts_channel: discord.TextChannel, - duration: float | None, - reason: str | None + duration: float, + reason: str ) -> None: """Invokes the command matching the infraction name.""" command_name = self.name.lower() @@ -81,10 +82,10 @@ async def invoke( ctx = FakeContext(channel) if self.name in ("KICK", "WARNING", "WATCH", "NOTE"): - await command(ctx, user, reason=reason) + await command(ctx, user, reason=reason or None) else: duration = arrow.utcnow() + timedelta(seconds=duration) if duration else None - await command(ctx, user, duration, reason=reason) + await command(ctx, user, duration, reason=reason or None) class InfractionAndNotification(ActionEntry): @@ -102,29 +103,31 @@ class InfractionAndNotification(ActionEntry): "the harsher one will be applied (by type or duration).\n\n" "Valid infraction types in order of harshness: " ) + ", ".join(infraction.name for infraction in Infraction), - "infraction_duration": "How long the infraction should last for in seconds, or 'None' for permanent.", + "infraction_duration": "How long the infraction should last for in seconds. 0 for permanent.", "infraction_reason": "The reason delivered with the infraction.", "infraction_channel": ( "The channel ID in which to invoke the infraction (and send the confirmation message). " - "If blank, the infraction will be sent in the context channel. If the ID fails to resolve, it will default " - "to the mod-alerts channel." + "If 0, the infraction will be sent in the context channel. If the ID otherwise fails to resolve, " + "it will default to the mod-alerts channel." ), "dm_content": "The contents of a message to be DMed to the offending user.", "dm_embed": "The contents of the embed to be DMed to the offending user." } - dm_content: str | None - dm_embed: str | None - infraction_type: Infraction | None - infraction_reason: str | None - infraction_duration: float | None - infraction_channel: int | None + dm_content: str + dm_embed: str + infraction_type: Infraction + infraction_reason: str + infraction_duration: float + infraction_channel: int @validator("infraction_type", pre=True) @classmethod - def convert_infraction_name(cls, infr_type: str) -> Infraction: + def convert_infraction_name(cls, infr_type: str | Infraction) -> Infraction: """Convert the string to an Infraction by name.""" - return Infraction[infr_type.replace(" ", "_").upper()] if infr_type else None + if isinstance(infr_type, Infraction): + return infr_type + return Infraction[infr_type.replace(" ", "_").upper()] async def action(self, ctx: FilterContext) -> None: """Send the notification to the user, and apply any specified infractions.""" @@ -150,7 +153,7 @@ async def action(self, ctx: FilterContext) -> None: except Forbidden: ctx.action_descriptions.append("failed to notify") - if self.infraction_type is not None: + if self.infraction_type != Infraction.NONE: alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) if self.infraction_channel: channel = bot_module.instance.get_channel(self.infraction_channel) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 9fc15410ef..17a9337837 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from enum import EnumMeta from functools import partial -from typing import Any, Callable, Coroutine, Optional, TypeVar, Union +from typing import Any, Callable, Coroutine, Optional, TypeVar import discord from botcore.site_api import ResponseCodeError @@ -114,24 +114,12 @@ def populate_embed_from_dict(embed: Embed, data: dict) -> None: embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) -def remove_optional(type_: type) -> tuple[bool, type]: - """Return whether the type is Optional, and the Union of types which aren't None.""" - if not hasattr(type_, "__args__"): - return False, type_ - args = list(type_.__args__) - if type(None) not in args: - return False, type_ - args.remove(type(None)) - return True, Union[tuple(args)] - - def parse_value(value: str, type_: type[T]) -> T: """Parse the value and attempt to convert it to the provided type.""" - is_optional, type_ = remove_optional(type_) - if is_optional and value == '""': - return None if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias type_ = type_.__origin__ + if value == '""': + return type_() if type_ in (tuple, list, set): return list(value.split(",")) if type_ is bool: @@ -273,7 +261,7 @@ def __init__(self, setting_name: str, update_callback: Callable): class FreeInputModal(discord.ui.Modal): """A modal to freely enter a value for a setting.""" - def __init__(self, setting_name: str, required: bool, type_: type, update_callback: Callable): + def __init__(self, setting_name: str, type_: type, update_callback: Callable): title = f"{setting_name} Input" if len(setting_name) < MAX_MODAL_TITLE_LENGTH - 6 else "Setting Input" super().__init__(timeout=COMPONENT_TIMEOUT, title=title) @@ -282,13 +270,16 @@ def __init__(self, setting_name: str, required: bool, type_: type, update_callba self.update_callback = update_callback label = setting_name if len(setting_name) < MAX_MODAL_TITLE_LENGTH else "Value" - self.setting_input = discord.ui.TextInput(label=label, style=discord.TextStyle.paragraph, required=required) + self.setting_input = discord.ui.TextInput(label=label, style=discord.TextStyle.paragraph, required=False) self.add_item(self.setting_input) async def on_submit(self, interaction: Interaction) -> None: """Update the setting with the new value in the embed.""" try: - value = self.type_(self.setting_input.value) + if not self.setting_input.value: + value = self.type_() + else: + value = self.type_(self.setting_input.value) except (ValueError, TypeError): await interaction.response.send_message( f"Could not process the input value for `{self.setting_name}`.", ephemeral=True @@ -436,7 +427,6 @@ async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.S """Prompt the user to give an override value for the setting they selected, and respond to the interaction.""" setting_name = select.values[0] type_ = self.type_per_setting_name[setting_name] - is_optional, type_ = remove_optional(type_) if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias type_ = type_.__origin__ new_view = self.copy() @@ -462,7 +452,7 @@ async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.S view = EnumSelectView(setting_name, type_, update_callback) await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) else: - await interaction.response.send_modal(FreeInputModal(setting_name, not is_optional, type_, update_callback)) + await interaction.response.send_modal(FreeInputModal(setting_name, type_, update_callback)) self.stop() @abstractmethod From a4172e1dce09c79387f283f4b4ea41e3aae16cc1 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 4 Nov 2022 14:27:33 +0200 Subject: [PATCH 067/132] Filter match now shows all triggers instead of messages The purpose of this command is to be able to find any relevant filters. It was not possible to find whitelist filters in the previous method, as there are no alerts for whitelist triggers. Instead, each filterlist now additionally returns what filters were triggered while processing the context. Additionally, a `no_user` argument was added to `filter match` to allow checking for triggers on one's own messages (otherwise there's almost no filtering on a mod's messages). --- bot/exts/filtering/_filter_lists/antispam.py | 12 +++--- bot/exts/filtering/_filter_lists/domain.py | 8 ++-- bot/exts/filtering/_filter_lists/extension.py | 12 +++--- .../filtering/_filter_lists/filter_list.py | 4 +- bot/exts/filtering/_filter_lists/invite.py | 19 ++++++---- bot/exts/filtering/_filter_lists/token.py | 8 ++-- bot/exts/filtering/_filter_lists/unique.py | 8 ++-- bot/exts/filtering/filtering.py | 37 ++++++++++++------- 8 files changed, 67 insertions(+), 41 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index b2f8730948..ed86c92c02 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -17,7 +17,7 @@ from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filter_lists.filter_list import ListType, SubscribingAtomicList, UniquesListBase from bot.exts.filtering._filters.antispam import antispam_filter_types -from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._filters.filter import Filter, UniqueFilter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction from bot.exts.filtering._ui.ui import build_mod_alert @@ -55,10 +55,12 @@ def get_filter_type(self, content: str) -> type[UniqueFilter] | None: self._already_warned.add(content) return None - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + async def actions_for( + self, ctx: FilterContext + ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" if not ctx.message: - return None, [] + return None, [], {} sublist: SubscribingAtomicList = self[ListType.DENY] potential_filters = [sublist.filters[id_] for id_ in sublist.subscriptions[ctx.event]] @@ -71,7 +73,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, new_ctx = ctx.replace(content=relevant_messages) triggers = await sublist.filter_list_result(new_ctx) if not triggers: - return None, [] + return None, [], {} if ctx.author not in self.message_deletion_queue: self.message_deletion_queue[ctx.author] = DeletionContext() @@ -99,7 +101,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, current_actions.pop("infraction_and_notification", None) # Provide some message in case another filter list wants there to be an alert. - return current_actions, ["Handling spam event..."] + return current_actions, ["Handling spam event..."], {ListType.DENY: triggers} def _create_deletion_context_handler(self, context_id: Member) -> Callable[[FilterContext], Coroutine]: async def schedule_processing(ctx: FilterContext) -> None: diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index 0b56e8d730..f4062edfe4 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -42,11 +42,13 @@ def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {DomainFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + async def actions_for( + self, ctx: FilterContext + ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" text = ctx.content if not text: - return None, [] + return None, [], {} text = clean_input(text) urls = {match.group(1).lower().rstrip("/") for match in URL_RE.finditer(text)} @@ -59,4 +61,4 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, if triggers: actions = self[ListType.DENY].merge_actions(triggers) messages = self[ListType.DENY].format_messages(triggers) - return actions, messages + return actions, messages, {ListType.DENY: triggers} diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index a53520bf71..a739d7191b 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -61,15 +61,17 @@ def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {ExtensionFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + async def actions_for( + self, ctx: FilterContext + ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" # Return early if the message doesn't have attachments. if not ctx.message or not ctx.message.attachments: - return None, [] + return None, [], {} _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) if failed: # There's no extension filtering in this context. - return None, [] + return None, [], {} # Find all extensions in the message. all_ext = { @@ -85,7 +87,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, not_allowed = {ext: filename for ext, filename in all_ext if ext not in allowed_ext} if not not_allowed: # Yes, it's a double negative. Meaning all attachments are allowed :) - return None, [] + return None, [], {ListType.ALLOW: triggered} # Something is disallowed. if ".py" in not_allowed: @@ -111,4 +113,4 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, ) ctx.matches += not_allowed.values() - return self[ListType.ALLOW].defaults.actions, [f"`{ext}`" for ext in not_allowed] + return self[ListType.ALLOW].defaults.actions, [f"`{ext}`" for ext in not_allowed], {ListType.ALLOW: triggered} diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 938766acae..f3761fbf98 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -184,7 +184,9 @@ def filter_types(self) -> set[type[T]]: """Return the types of filters used by this list.""" @abstractmethod - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + async def actions_for( + self, ctx: FilterContext + ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" def _create_filter(self, filter_data: dict, defaults: Defaults) -> T | None: diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 911b951dd1..36031f2768 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -48,7 +48,9 @@ def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {InviteFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + async def actions_for( + self, ctx: FilterContext + ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" text = clean_input(ctx.content) @@ -58,7 +60,8 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, matches = list(DISCORD_INVITE.finditer(text)) invite_codes = {m.group("invite") for m in matches} if not invite_codes: - return None, [] + return None, [], {} + all_triggers = {} _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) # If the allowed list doesn't operate in the context, unknown invites are allowed. @@ -99,16 +102,17 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, if check_if_allowed: # Whether unknown invites need to be checked. new_ctx = ctx.replace(content=guilds_for_inspection) - allowed = { - filter_.content for filter_ in self[ListType.ALLOW].filters.values() + all_triggers[ListType.ALLOW] = [ + filter_ for filter_ in self[ListType.ALLOW].filters.values() if await filter_.triggered_on(new_ctx) - } + ] + allowed = {filter_.content for filter_ in all_triggers[ListType.ALLOW]} unknown_invites.update({ code: invite for code, invite in invites_for_inspection.items() if invite.guild.id not in allowed }) if not triggered and not unknown_invites: - return None, [] + return None, [], all_triggers actions = None if unknown_invites: # There are invites which weren't allowed but aren't explicitly blocked. @@ -119,6 +123,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, actions |= self[ListType.DENY].merge_actions(triggered) else: actions = self[ListType.DENY].merge_actions(triggered) + all_triggers[ListType.DENY] = triggered blocked_invites |= unknown_invites ctx.matches += {match[0] for match in matches if match.group("invite") in blocked_invites} @@ -127,7 +132,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, messages += [ f"`{code} - {invite.guild.id}`" if invite else f"`{code}`" for code, invite in unknown_invites.items() ] - return actions, messages + return actions, messages, all_triggers @staticmethod def _guild_embed(invite: Invite) -> Embed: diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index 274dc5ea75..e4dbf47179 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -43,11 +43,13 @@ def filter_types(self) -> set[type[Filter]]: """Return the types of filters used by this list.""" return {TokenFilter} - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + async def actions_for( + self, ctx: FilterContext + ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" text = ctx.content if not text: - return None, [] + return None, [], {} if SPOILER_RE.search(text): text = self._expand_spoilers(text) text = clean_input(text) @@ -59,7 +61,7 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, if triggers: actions = self[ListType.DENY].merge_actions(triggers) messages = self[ListType.DENY].format_messages(triggers) - return actions, messages + return actions, messages, {ListType.DENY: triggers} @staticmethod def _expand_spoilers(text: str) -> str: diff --git a/bot/exts/filtering/_filter_lists/unique.py b/bot/exts/filtering/_filter_lists/unique.py index ecc49af870..2cc1b78b2b 100644 --- a/bot/exts/filtering/_filter_lists/unique.py +++ b/bot/exts/filtering/_filter_lists/unique.py @@ -2,7 +2,7 @@ from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filter_lists.filter_list import ListType, UniquesListBase -from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._filters.filter import Filter, UniqueFilter from bot.exts.filtering._filters.unique import unique_filter_types from bot.exts.filtering._settings import ActionSettings @@ -29,7 +29,9 @@ def get_filter_type(self, content: str) -> type[UniqueFilter] | None: self._already_warned.add(content) return None - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + async def actions_for( + self, ctx: FilterContext + ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" triggers = await self[ListType.DENY].filter_list_result(ctx) actions = None @@ -37,4 +39,4 @@ async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, if triggers: actions = self[ListType.DENY].merge_actions(triggers) messages = self[ListType.DENY].format_messages(triggers) - return actions, messages + return actions, messages, {ListType.DENY: triggers} diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 34ca9f6e5e..bd1345bc0c 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -175,7 +175,7 @@ async def on_message(self, msg: Message) -> None: self.message_cache.append(msg) ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) - result_actions, list_messages = await self._resolve_action(ctx) + result_actions, list_messages, _ = await self._resolve_action(ctx) if result_actions: await result_actions.action(ctx) if ctx.send_alert: @@ -498,31 +498,37 @@ async def setting(self, ctx: Context, setting_name: str | None) -> None: await ctx.send(embed=embed) @filter.command(name="match") - async def f_match(self, ctx: Context, message: Message | None, *, string: str | None) -> None: + async def f_match( + self, ctx: Context, no_user: bool | None, message: Message | None, *, string: str | None + ) -> None: """ Post any responses from the filter lists for the given message or string. - If there's a message the string will be ignored. Note that if a message is provided, it will go through all - validations appropriate to where it was sent and who sent it. + If there's a `message`, the `string` will be ignored. Note that if a `message` is provided, it will go through + all validations appropriate to where it was sent and who sent it. To check for matches regardless of the author + (for example if the message was sent by another staff member or yourself) set `no_user` to '1' or 'True'. - If a string is provided, it will be validated in the context of a user with no roles in python-general. + If a `string` is provided, it will be validated in the context of a user with no roles in python-general. """ if not message and not string: - raise BadArgument(":x: Please provide input.") + raise BadArgument("Please provide input.") if message: + user = None if no_user else message.author filter_ctx = FilterContext( - Event.MESSAGE, message.author, message.channel, message.content, message, message.embeds + Event.MESSAGE, user, message.channel, message.content, message, message.embeds ) else: filter_ctx = FilterContext( Event.MESSAGE, None, ctx.guild.get_channel(Channels.python_general), string, None ) - _, list_messages = await self._resolve_action(filter_ctx) + _, _, triggers = await self._resolve_action(filter_ctx) lines = [] - for filter_list, list_message_list in list_messages.items(): - if list_message_list: - lines.extend([f"**{filter_list.name.title()}s**", *list_message_list, "\n"]) + for filter_list, list_triggers in triggers.items(): + for sublist_type, sublist_triggers in list_triggers.items(): + if sublist_triggers: + triggers_repr = map(str, sublist_triggers) + lines.extend([f"**{filter_list[sublist_type].label.title()}s**", *triggers_repr, "\n"]) lines = lines[:-1] # Remove last newline. embed = Embed(colour=Colour.blue(), title="Match results") @@ -745,7 +751,9 @@ def _load_raw_filter_list(self, list_data: dict) -> AtomicList | None: self.filter_lists[list_name] = filter_list_types[list_name](self) return self.filter_lists[list_name].add_list(list_data) - async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSettings], dict[FilterList, list[str]]]: + async def _resolve_action( + self, ctx: FilterContext + ) -> tuple[Optional[ActionSettings], dict[FilterList, list[str]], dict[FilterList, dict[ListType, list[Filter]]]]: """ Return the actions that should be taken for all filter lists in the given context. @@ -754,8 +762,9 @@ async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSett """ actions = [] messages = {} + triggers = {} for filter_list in self._subscriptions[ctx.event]: - list_actions, list_message = await filter_list.actions_for(ctx) + list_actions, list_message, triggers[filter_list] = await filter_list.actions_for(ctx) if list_actions: actions.append(list_actions) if list_message: @@ -765,7 +774,7 @@ async def _resolve_action(self, ctx: FilterContext) -> tuple[Optional[ActionSett if actions: result_actions = reduce(operator.or_, (action for action in actions)) - return result_actions, messages + return result_actions, messages, triggers async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, list[str]]) -> None: """Build an alert message from the filter context, and send it via the alert webhook.""" From 5cecb3be45dbd2488c75f12b0f587ddf0db0b767 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 4 Nov 2022 16:24:00 +0200 Subject: [PATCH 068/132] Edit description on filter add/edit, bug fixes `process_content` was changed to `process_input` to also edit the description when necessary. Fixes unhandled filter_type being None on filter add/edit. Moves the missing filter type warning to a more appropriate location to not trigger during filter add/edit. --- bot/exts/filtering/_filter_lists/filter_list.py | 13 ++++++++----- bot/exts/filtering/_filter_lists/unique.py | 3 --- bot/exts/filtering/_filters/domain.py | 8 ++++---- bot/exts/filtering/_filters/extension.py | 8 ++++---- bot/exts/filtering/_filters/filter.py | 6 +++--- bot/exts/filtering/_filters/invite.py | 12 ++++++++---- bot/exts/filtering/_filters/token.py | 8 ++++---- bot/exts/filtering/_ui/filter.py | 9 +++++++++ bot/exts/filtering/filtering.py | 7 +++++-- 9 files changed, 45 insertions(+), 29 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index f3761fbf98..9ae45bfaf0 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -152,6 +152,8 @@ class FilterList(dict[ListType, AtomicList], typing.Generic[T], FieldRequiring): # Names must be unique across all filter lists. name = FieldRequiring.MUST_SET_UNIQUE + _already_warned = set() + def add_list(self, list_data: dict) -> AtomicList: """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" actions, validations = create_settings(list_data["settings"], keep_empty=True) @@ -192,11 +194,14 @@ async def actions_for( def _create_filter(self, filter_data: dict, defaults: Defaults) -> T | None: """Create a filter from the given data.""" try: - filter_type = self.get_filter_type(filter_data["content"]) + content = filter_data["content"] + filter_type = self.get_filter_type(content) if filter_type: return filter_type(filter_data, defaults) - else: - return None + elif content not in self._already_warned: + log.warn(f"A filter named {content} was supplied, but no matching implementation found.") + self._already_warned.add(content) + return None except TypeError as e: log.warning(e) @@ -240,8 +245,6 @@ class UniquesListBase(FilterList[UniqueFilter]): Each unique filter subscribes to a subset of events to respond to. """ - _already_warned = set() - def __init__(self, filtering_cog: 'Filtering'): super().__init__() self.filtering_cog = filtering_cog diff --git a/bot/exts/filtering/_filter_lists/unique.py b/bot/exts/filtering/_filter_lists/unique.py index 2cc1b78b2b..fdf22e3b60 100644 --- a/bot/exts/filtering/_filter_lists/unique.py +++ b/bot/exts/filtering/_filter_lists/unique.py @@ -24,9 +24,6 @@ def get_filter_type(self, content: str) -> type[UniqueFilter] | None: try: return unique_filter_types[content] except KeyError: - if content not in self._already_warned: - log.warn(f"A unique filter named {content} was supplied, but no matching implementation found.") - self._already_warned.add(content) return None async def actions_for( diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 4cc3a6f5ae..7c229fdcb3 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -50,13 +50,13 @@ async def triggered_on(self, ctx: FilterContext) -> bool: return False @classmethod - async def process_content(cls, content: str) -> str: + async def process_input(cls, content: str, description: str) -> tuple[str, str]: """ - Process the content into a form which will work with the filtering. + Process the content and description into a form which will work with the filtering. - A ValueError should be raised if the content can't be used. + A BadArgument should be raised if the content can't be used. """ match = URL_RE.fullmatch(content) if not match or not match.group(1): raise BadArgument(f"`{content}` is not a URL.") - return match.group(1) + return match.group(1), description diff --git a/bot/exts/filtering/_filters/extension.py b/bot/exts/filtering/_filters/extension.py index f3f64532f7..97eddc406d 100644 --- a/bot/exts/filtering/_filters/extension.py +++ b/bot/exts/filtering/_filters/extension.py @@ -16,12 +16,12 @@ async def triggered_on(self, ctx: FilterContext) -> bool: return self.content in ctx.content @classmethod - async def process_content(cls, content: str) -> str: + async def process_input(cls, content: str, description: str) -> tuple[str, str]: """ - Process the content into a form which will work with the filtering. + Process the content and description into a form which will work with the filtering. - A ValueError should be raised if the content can't be used. + A BadArgument should be raised if the content can't be used. """ if not content.startswith("."): content = f".{content}" - return content + return content, description diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 4ae7ec45f1..45b571b54b 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -65,13 +65,13 @@ def validate_filter_settings(cls, extra_fields: dict) -> tuple[bool, str | None] return True, None @classmethod - async def process_content(cls, content: str) -> str: + async def process_input(cls, content: str, description: str) -> tuple[str, str]: """ - Process the content into a form which will work with the filtering. + Process the content and description into a form which will work with the filtering. A BadArgument should be raised if the content can't be used. """ - return content + return content, description def __str__(self) -> str: """A string representation of the filter.""" diff --git a/bot/exts/filtering/_filters/invite.py b/bot/exts/filtering/_filters/invite.py index e8f3e98519..f68adbb23b 100644 --- a/bot/exts/filtering/_filters/invite.py +++ b/bot/exts/filtering/_filters/invite.py @@ -25,11 +25,11 @@ async def triggered_on(self, ctx: FilterContext) -> bool: return self.content in ctx.content @classmethod - async def process_content(cls, content: str) -> str: + async def process_input(cls, content: str, description: str) -> tuple[str, str]: """ - Process the content into a form which will work with the filtering. + Process the content and description into a form which will work with the filtering. - A ValueError should be raised if the content can't be used. + A BadArgument should be raised if the content can't be used. """ match = DISCORD_INVITE.fullmatch(content) if not match or not match.group("invite"): @@ -41,4 +41,8 @@ async def process_content(cls, content: str) -> str: raise BadArgument(f"`{invite_code}` is not a valid Discord invite code.") if not invite.guild: raise BadArgument("Did you just try to add a group DM?") - return str(invite.guild.id) + + guild_name = invite.guild.name if hasattr(invite.guild, "name") else "" + if guild_name.lower() not in description.lower(): + description = " - ".join(part for part in (f'Guild "{guild_name}"', description) if part) + return str(invite.guild.id), description diff --git a/bot/exts/filtering/_filters/token.py b/bot/exts/filtering/_filters/token.py index f61d388463..3cd9b909d1 100644 --- a/bot/exts/filtering/_filters/token.py +++ b/bot/exts/filtering/_filters/token.py @@ -22,14 +22,14 @@ async def triggered_on(self, ctx: FilterContext) -> bool: return False @classmethod - async def process_content(cls, content: str) -> str: + async def process_input(cls, content: str, description: str) -> tuple[str, str]: """ - Process the content into a form which will work with the filtering. + Process the content and description into a form which will work with the filtering. - A ValueError should be raised if the content can't be used. + A BadArgument should be raised if the content can't be used. """ try: re.compile(content) except re.error as e: raise BadArgument(str(e)) - return content + return content, description diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index 37584e9fd6..e41c663c58 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -255,7 +255,16 @@ async def update_embed( """ if content is not None or description is not None: if content is not None: + filter_type = self.filter_list.get_filter_type(content) + if not filter_type: + if isinstance(interaction_or_msg, discord.Message): + send_method = interaction_or_msg.channel.send + else: + send_method = interaction_or_msg.response.send_message + await send_method(f":x: Could not find a filter type appropriate for `{content}`.") + return self.content = content + self.filter_type = filter_type else: content = self.content # If there's no content or description, use the existing values. if description is self._REMOVE: diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index bd1345bc0c..a204e53f38 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -857,6 +857,9 @@ async def _add_filter( ) -> None: """Add a filter to the database.""" filter_type = filter_list.get_filter_type(content) + if not filter_type: + await ctx.reply(f":x: Could not find a filter type appropriate for `{content}`.") + return description, settings, filter_settings = description_and_settings_converter( filter_list, list_type, @@ -937,7 +940,7 @@ async def _post_new_filter( if not valid: raise BadArgument(f"Error while validating filter-specific settings: {error_msg}") - content = await filter_type.process_content(content) + content, description = await filter_type.process_input(content, description) list_id = filter_list[list_type].id description = description or None @@ -971,7 +974,7 @@ async def _patch_filter( raise BadArgument(f"Error while validating filter-specific settings: {error_msg}") if content != filter_.content: - content = await filter_type.process_content(content) + content, description = await filter_type.process_input(content, description) # If the setting is not in `settings`, the override was either removed, or there wasn't one in the first place. for current_settings in (filter_.actions, filter_.validations): From 7fcec400d3b9d5e28222ee83e5fa0a48038ad4e4 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 5 Nov 2022 15:15:32 +0200 Subject: [PATCH 069/132] Bring back auto-infraction reporting - When a filter is set to infract, it will send a message in #filter-log (See PR #1893) - Once a week the cog will send a message of all auto-infractions sent in the last 7 days, with a way to trigger it manually (See PR #2267) --- .../filtering/_filter_lists/filter_list.py | 23 +++- bot/exts/filtering/_filters/filter.py | 3 + bot/exts/filtering/filtering.py | 101 +++++++++++++++++- 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 9ae45bfaf0..fd243a1098 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -9,6 +9,7 @@ from operator import or_ from typing import Any +import arrow from discord.ext.commands import BadArgument from bot.exts.filtering._filter_context import Event, FilterContext @@ -55,6 +56,8 @@ class AtomicList: """ id: int + created_at: arrow.Arrow + updated_at: arrow.Arrow name: str list_type: ListType defaults: Defaults @@ -166,7 +169,15 @@ def add_list(self, list_data: dict) -> AtomicList: if new_filter: filters[filter_data["id"]] = new_filter - self[list_type] = AtomicList(list_data["id"], self.name, list_type, defaults, filters) + self[list_type] = AtomicList( + list_data["id"], + arrow.get(list_data["created_at"]), + arrow.get(list_data["updated_at"]), + self.name, + list_type, + defaults, + filters + ) return self[list_type] def add_filter(self, list_type: ListType, filter_data: dict) -> T | None: @@ -255,7 +266,15 @@ def add_list(self, list_data: dict) -> SubscribingAtomicList: actions, validations = create_settings(list_data["settings"], keep_empty=True) list_type = ListType(list_data["list_type"]) defaults = Defaults(actions, validations) - new_list = SubscribingAtomicList(list_data["id"], self.name, list_type, defaults, {}) + new_list = SubscribingAtomicList( + list_data["id"], + arrow.get(list_data["created_at"]), + arrow.get(list_data["updated_at"]), + self.name, + list_type, + defaults, + {} + ) self[list_type] = new_list filters = {} diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 45b571b54b..49c163d993 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,6 +1,7 @@ from abc import abstractmethod from typing import Any +import arrow from pydantic import ValidationError from bot.exts.filtering._filter_context import Event, FilterContext @@ -26,6 +27,8 @@ def __init__(self, filter_data: dict, defaults: Defaults | None = None): self.id = filter_data["id"] self.content = filter_data["content"] self.description = filter_data["description"] + self.created_at = arrow.get(filter_data["created_at"]) + self.updated_at = arrow.get(filter_data["updated_at"]) self.actions, self.validations = create_settings(filter_data["settings"], defaults=defaults) if self.extra_fields_type: self.extra_fields = self.extra_fields_type.parse_raw(filter_data["additional_field"] or "{}") # noqa: P103 diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index a204e53f38..c4c118b6f8 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1,3 +1,4 @@ +import datetime import json import operator import re @@ -6,11 +7,12 @@ from io import BytesIO from typing import Literal, Optional, get_type_hints +import arrow import discord from botcore.site_api import ResponseCodeError from discord import Colour, Embed, HTTPException, Message, MessageType -from discord.ext import commands -from discord.ext.commands import BadArgument, Cog, Context, has_any_role +from discord.ext import commands, tasks +from discord.ext.commands import BadArgument, Cog, Context, command, has_any_role import bot import bot.exts.filtering._ui.filter as filters_ui @@ -22,6 +24,7 @@ from bot.exts.filtering._filter_lists.filter_list import AtomicList from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings import ActionSettings +from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction from bot.exts.filtering._ui.filter import ( build_filter_repr_dict, description_and_settings_converter, filter_serializable_overrides, populate_embed_from_dict ) @@ -33,11 +36,13 @@ from bot.exts.filtering._utils import past_tense, repr_equals, starting_value, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator +from bot.utils.channel import is_mod_channel from bot.utils.message_cache import MessageCache log = get_logger(__name__) CACHE_SIZE = 100 +WEEKLY_REPORT_ISO_DAY = 3 # 1=Monday, 7=Sunday class Filtering(Cog): @@ -81,6 +86,7 @@ async def cog_load(self) -> None: log.error(f"Failed to fetch filters webhook with ID `{Webhooks.filters}`.") self.collect_loaded_types(example_list) + self.weekly_auto_infraction_report_task.start() def subscribe(self, filter_list: FilterList, *events: Event) -> None: """ @@ -734,6 +740,14 @@ async def delete_list() -> None: view=DeleteConfirmationView(ctx.author, delete_list) ) + # endregion + # region: utility commands + + @command(name="filter_report") + async def force_send_weekly_report(self, ctx: Context) -> None: + """Respond with a list of auto-infractions added in the last 7 days.""" + await self.send_weekly_auto_infraction_report(ctx.channel) + # endregion # region: helper functions @@ -925,7 +939,30 @@ def _identical_filters_message(content: str, filter_list: FilterList, list_type: return msg @staticmethod + async def _maybe_alert_auto_infraction( + filter_list: FilterList, list_type: ListType, filter_: Filter, old_filter: Filter | None = None + ) -> None: + """If the filter is new and applies an auto-infraction, or was edited to apply a different one, log it.""" + infraction_type = filter_.overrides[0].get("infraction_type") + if not infraction_type: + infraction_type = filter_list[list_type].defaults.actions.get_setting("infraction_type") + if old_filter: + old_infraction_type = old_filter.overrides[0].get("infraction_type") + if not old_infraction_type: + old_infraction_type = filter_list[list_type].defaults.actions.get_setting("infraction_type") + if infraction_type == old_infraction_type: + return + + if infraction_type != Infraction.NONE: + filter_log = bot.instance.get_channel(Channels.filter_log) + if filter_log: + await filter_log.send( + f":warning: Heads up! The new {filter_list[list_type].label} filter " + f"({filter_}) will automatically {infraction_type.name.lower()} users." + ) + async def _post_new_filter( + self, msg: Message, filter_list: FilterList, list_type: ListType, @@ -951,13 +988,14 @@ async def _post_new_filter( response = await bot.instance.api_client.post('bot/filter/filters', json=to_serializable(payload)) new_filter = filter_list.add_filter(list_type, response) if new_filter: + await self._maybe_alert_auto_infraction(filter_list, list_type, new_filter) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, new_filter) await msg.reply(f"✅ Added filter: {new_filter}" + extra_msg) else: await msg.reply(":x: Could not create the filter. Are you sure it's implemented?") - @staticmethod async def _patch_filter( + self, filter_: Filter, msg: Message, filter_list: FilterList, @@ -991,6 +1029,7 @@ async def _patch_filter( ) # Return type can be None, but if it's being edited then it's not supposed to be. edited_filter = filter_list.add_filter(list_type, response) + await self._maybe_alert_auto_infraction(filter_list, list_type, edited_filter, filter_) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, edited_filter) await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) @@ -1078,6 +1117,62 @@ async def _search_filters( await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False, reply=True) # endregion + # region: tasks + + @tasks.loop(time=datetime.time(hour=18)) + async def weekly_auto_infraction_report_task(self) -> None: + """Trigger an auto-infraction report to be sent if it is the desired day of the week (WEEKLY_REPORT_ISO_DAY).""" + if arrow.utcnow().isoweekday() != WEEKLY_REPORT_ISO_DAY: + return + + await self.send_weekly_auto_infraction_report() + + async def send_weekly_auto_infraction_report(self, channel: discord.TextChannel | discord.Thread = None) -> None: + """ + Send a list of auto-infractions added in the last 7 days to the specified channel. + + If `channel` is not specified, it is sent to #mod-meta. + """ + seven_days_ago = arrow.utcnow().shift(days=-7) + if not channel: + log.info("Auto-infraction report: the channel to report to is missing.") + channel = self.bot.get_channel(Channels.mod_meta) + elif not is_mod_channel(channel): + # Silently fail if output is going to be a non-mod channel. + log.info(f"Auto-infraction report: the channel {channel} is not a mod channel.") + return + + found_filters = defaultdict(list) + # Extract all auto-infraction filters added in the past 7 days from each filter type + for filter_list in self.filter_lists.values(): + for sublist in filter_list.values(): + default_infraction_type = sublist.defaults.actions.get_setting("infraction_type") + for filter_ in sublist.filters.values(): + if max(filter_.created_at, filter_.updated_at) < seven_days_ago: + continue + infraction_type = filter_.overrides[0].get("infraction_type") + if ( + (infraction_type and infraction_type != Infraction.NONE) + or (not infraction_type and default_infraction_type != Infraction.NONE) + ): + found_filters[sublist.label].append((filter_, infraction_type or default_infraction_type)) + + # Nicely format the output so each filter list type is grouped + lines = [f"**Auto-infraction filters added since {seven_days_ago.format('YYYY-MM-DD')}**"] + for list_label, filters in found_filters.items(): + lines.append("\n".join([f"**{list_label.title()}**"]+[f"{filter_} ({infr})" for filter_, infr in filters])) + + if len(lines) == 1: + lines.append("Nothing to show") + + await channel.send("\n\n".join(lines)) + log.info("Successfully sent auto-infraction report.") + + # endregion + + async def cog_unload(self) -> None: + """Cancel the weekly auto-infraction filter report on cog unload.""" + self.weekly_auto_infraction_report_task.cancel() async def setup(bot: Bot) -> None: From e83717b8d6770a1562f2e82871a09b63c7b0c9b9 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 5 Nov 2022 16:06:47 +0200 Subject: [PATCH 070/132] Handle threads in channel_scope --- bot/exts/filtering/_settings_types/validations/channel_scope.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index ae6c24c97a..80f837a151 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -53,6 +53,8 @@ def triggers_on(self, ctx: FilterContext) -> bool: if not hasattr(channel, "category"): # This is not a guild channel, outside the scope of this setting. return True + if hasattr(channel, "parent"): + channel = channel.parent enabled_channel = channel.id in self.enabled_channels or channel.name in self.enabled_channels disabled_channel = channel.id in self.disabled_channels or channel.name in self.disabled_channels From 48a892fa42e5efe971ab69340fc5d731623020e5 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 5 Nov 2022 22:49:57 +0200 Subject: [PATCH 071/132] Add message edit filtering This edit handler takes into account filters already triggered for the message and ignores them (as long as it's a denied type) To that end the message cache can now hold metadata to accompany each message in the cache. --- bot/exts/filtering/_filter_context.py | 11 ++++ bot/exts/filtering/_filter_lists/antispam.py | 2 +- .../filtering/_filter_lists/filter_list.py | 18 ++++-- .../filtering/_filters/unique/rich_embed.py | 6 ++ bot/exts/filtering/filtering.py | 55 +++++++++++++------ bot/utils/message_cache.py | 23 ++++++-- 6 files changed, 87 insertions(+), 28 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 3227b333ae..4a213535ad 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -7,6 +7,8 @@ from discord import DMChannel, Member, Message, TextChannel, Thread, User +from bot.utils.message_cache import MessageCache + if typing.TYPE_CHECKING: from bot.exts.filtering._filters.filter import Filter @@ -29,6 +31,8 @@ class FilterContext: content: str | Iterable # What actually needs filtering message: Message | None # The message involved embeds: list = field(default_factory=list) # Any embeds involved + before_message: Message | None = None + message_cache: MessageCache | None = None # Output context dm_content: str = field(default_factory=str) # The content to DM the invoker dm_embed: str = field(default_factory=str) # The embed description to DM the invoker @@ -45,6 +49,13 @@ class FilterContext: related_channels: set[TextChannel | Thread | DMChannel] = field(default_factory=set) attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. + @classmethod + def from_message( + cls, event: Event, message: Message, before: Message | None = None, cache: MessageCache | None = None + ) -> FilterContext: + """Create a filtering context from the attributes of a message.""" + return cls(event, message.author, message.channel, message.content, message, message.embeds, before, cache) + def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" return replace(self, **changes) diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index ed86c92c02..cf58757238 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -68,7 +68,7 @@ async def actions_for( earliest_relevant_at = arrow.utcnow() - timedelta(seconds=max_interval) relevant_messages = list( - takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.filtering_cog.message_cache) + takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.message_cache) ) new_ctx = ctx.replace(content=relevant_messages) triggers = await sublist.filter_list_result(new_ctx) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index fd243a1098..b5d6141d75 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -47,7 +47,8 @@ def list_type_converter(argument: str) -> ListType: raise BadArgument(f"No matching list type found for {argument!r}.") -@dataclass(frozen=True) +# AtomicList and its subclasses must have eq=False, otherwise the dataclass deco will replace the hash function. +@dataclass(frozen=True, eq=False) class AtomicList: """ Represents the atomic structure of a single filter list as it appears in the database. @@ -84,9 +85,8 @@ async def filter_list_result(self, ctx: FilterContext) -> list[Filter]: """ return await self._create_filter_list_result(ctx, self.defaults, self.filters.values()) - @staticmethod async def _create_filter_list_result( - ctx: FilterContext, defaults: Defaults, filters: Iterable[Filter] + self, ctx: FilterContext, defaults: Defaults, filters: Iterable[Filter] ) -> list[Filter]: """A helper function to evaluate the result of `filter_list_result`.""" passed_by_default, failed_by_default = defaults.validations.evaluate(ctx) @@ -103,6 +103,13 @@ async def _create_filter_list_result( if await filter_.triggered_on(ctx): relevant_filters.append(filter_) + if ctx.event == Event.MESSAGE_EDIT and ctx.message and self.list_type == ListType.DENY: + previously_triggered = ctx.message_cache.get_message_metadata(ctx.message.id) + if previously_triggered and self in previously_triggered: + ignore_filters = previously_triggered[self] + # This updates the cache. Some filters are ignored, but they're necessary if there's another edit. + previously_triggered[self] = relevant_filters + relevant_filters = [filter_ for filter_ in relevant_filters if filter_ not in ignore_filters] return relevant_filters def default(self, setting_name: str) -> Any: @@ -144,6 +151,9 @@ def format_messages(triggers: list[Filter], *, expand_single_filter: bool = True messages = [f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers] return messages + def __hash__(self): + return hash(id(self)) + T = typing.TypeVar("T", bound=Filter) @@ -220,7 +230,7 @@ def __hash__(self): return hash(id(self)) -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class SubscribingAtomicList(AtomicList): """ A base class for a list of unique filters. diff --git a/bot/exts/filtering/_filters/unique/rich_embed.py b/bot/exts/filtering/_filters/unique/rich_embed.py index 09d5133739..5c3517e102 100644 --- a/bot/exts/filtering/_filters/unique/rich_embed.py +++ b/bot/exts/filtering/_filters/unique/rich_embed.py @@ -20,6 +20,12 @@ class RichEmbedFilter(UniqueFilter): async def triggered_on(self, ctx: FilterContext) -> bool: """Determine if `msg` contains any rich embeds not auto-generated from a URL.""" if ctx.embeds: + if ctx.event == Event.MESSAGE_EDIT: + # If the edit delta is less than 100 microseconds, it's probably a double filter trigger. + delta = ctx.message.edited_at - (ctx.before_message.edited_at or ctx.before_message.created_at) + if delta.total_seconds() < 0.0001: + return False + for embed in ctx.embeds: if embed.type == "rich": urls = URL_RE.findall(ctx.content) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index c4c118b6f8..05b2339b95 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -180,8 +180,30 @@ async def on_message(self, msg: Message) -> None: return self.message_cache.append(msg) - ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) - result_actions, list_messages, _ = await self._resolve_action(ctx) + ctx = FilterContext.from_message(Event.MESSAGE, msg, None, self.message_cache) + result_actions, list_messages, triggers = await self._resolve_action(ctx) + self.message_cache.update(msg, metadata=triggers) + if result_actions: + await result_actions.action(ctx) + if ctx.send_alert: + await self._send_alert(ctx, list_messages) + + @Cog.listener() + async def on_message_edit(self, before: discord.Message, after: discord.Message) -> None: + """Filter the contents of an edited message. Don't reinvoke filters already invoked on the `before` version.""" + # Only check changes to the message contents/attachments and embed additions, not pin status etc. + if all(( + before.content == after.content, # content hasn't changed + before.attachments == after.attachments, # attachments haven't changed + len(before.embeds) >= len(after.embeds) # embeds haven't been added + )): + return + + # Update the cache first, it might be used by the antispam filter. + # No need to update the triggers, they're going to be updated inside the sublists if necessary. + self.message_cache.update(after) + ctx = FilterContext.from_message(Event.MESSAGE_EDIT, after, before, self.message_cache) + result_actions, list_messages, triggers = await self._resolve_action(ctx) if result_actions: await result_actions.action(ctx) if ctx.send_alert: @@ -520,21 +542,17 @@ async def f_match( raise BadArgument("Please provide input.") if message: user = None if no_user else message.author - filter_ctx = FilterContext( - Event.MESSAGE, user, message.channel, message.content, message, message.embeds - ) + filter_ctx = FilterContext(Event.MESSAGE, user, message.channel, message.content, message, message.embeds) else: - filter_ctx = FilterContext( - Event.MESSAGE, None, ctx.guild.get_channel(Channels.python_general), string, None - ) + python_general = ctx.guild.get_channel(Channels.python_general) + filter_ctx = FilterContext(Event.MESSAGE, None, python_general, string, None) _, _, triggers = await self._resolve_action(filter_ctx) lines = [] - for filter_list, list_triggers in triggers.items(): - for sublist_type, sublist_triggers in list_triggers.items(): - if sublist_triggers: - triggers_repr = map(str, sublist_triggers) - lines.extend([f"**{filter_list[sublist_type].label.title()}s**", *triggers_repr, "\n"]) + for sublist, sublist_triggers in triggers.items(): + if sublist_triggers: + triggers_repr = map(str, sublist_triggers) + lines.extend([f"**{sublist.label.title()}s**", *triggers_repr, "\n"]) lines = lines[:-1] # Remove last newline. embed = Embed(colour=Colour.blue(), title="Match results") @@ -767,7 +785,7 @@ def _load_raw_filter_list(self, list_data: dict) -> AtomicList | None: async def _resolve_action( self, ctx: FilterContext - ) -> tuple[Optional[ActionSettings], dict[FilterList, list[str]], dict[FilterList, dict[ListType, list[Filter]]]]: + ) -> tuple[ActionSettings | None, dict[FilterList, list[str]], dict[AtomicList, list[Filter]]]: """ Return the actions that should be taken for all filter lists in the given context. @@ -778,7 +796,8 @@ async def _resolve_action( messages = {} triggers = {} for filter_list in self._subscriptions[ctx.event]: - list_actions, list_message, triggers[filter_list] = await filter_list.actions_for(ctx) + list_actions, list_message, list_triggers = await filter_list.actions_for(ctx) + triggers.update({filter_list[list_type]: filters for list_type, filters in list_triggers.items()}) if list_actions: actions.append(list_actions) if list_message: @@ -945,11 +964,11 @@ async def _maybe_alert_auto_infraction( """If the filter is new and applies an auto-infraction, or was edited to apply a different one, log it.""" infraction_type = filter_.overrides[0].get("infraction_type") if not infraction_type: - infraction_type = filter_list[list_type].defaults.actions.get_setting("infraction_type") + infraction_type = filter_list[list_type].default("infraction_type") if old_filter: old_infraction_type = old_filter.overrides[0].get("infraction_type") if not old_infraction_type: - old_infraction_type = filter_list[list_type].defaults.actions.get_setting("infraction_type") + old_infraction_type = filter_list[list_type].default("infraction_type") if infraction_type == old_infraction_type: return @@ -1146,7 +1165,7 @@ async def send_weekly_auto_infraction_report(self, channel: discord.TextChannel # Extract all auto-infraction filters added in the past 7 days from each filter type for filter_list in self.filter_lists.values(): for sublist in filter_list.values(): - default_infraction_type = sublist.defaults.actions.get_setting("infraction_type") + default_infraction_type = sublist.default("infraction_type") for filter_ in sublist.filters.values(): if max(filter_.created_at, filter_.updated_at) < seven_days_ago: continue diff --git a/bot/utils/message_cache.py b/bot/utils/message_cache.py index f68d280c97..3e77e6a50e 100644 --- a/bot/utils/message_cache.py +++ b/bot/utils/message_cache.py @@ -31,20 +31,23 @@ def __init__(self, maxlen: int, *, newest_first: bool = False): self._start = 0 self._end = 0 - self._messages: list[t.Optional[Message]] = [None] * self.maxlen + self._messages: list[Message | None] = [None] * self.maxlen self._message_id_mapping = {} + self._message_metadata = {} - def append(self, message: Message) -> None: + def append(self, message: Message, *, metadata: dict | None = None) -> None: """Add the received message to the cache, depending on the order of messages defined by `newest_first`.""" if self.newest_first: self._appendleft(message) else: self._appendright(message) + self._message_metadata[message.id] = metadata def _appendright(self, message: Message) -> None: """Add the received message to the end of the cache.""" if self._is_full(): del self._message_id_mapping[self._messages[self._start].id] + del self._message_metadata[self._messages[self._start].id] self._start = (self._start + 1) % self.maxlen self._messages[self._end] = message @@ -56,6 +59,7 @@ def _appendleft(self, message: Message) -> None: if self._is_full(): self._end = (self._end - 1) % self.maxlen del self._message_id_mapping[self._messages[self._end].id] + del self._message_metadata[self._messages[self._end].id] self._start = (self._start - 1) % self.maxlen self._messages[self._start] = message @@ -69,6 +73,7 @@ def pop(self) -> Message: self._end = (self._end - 1) % self.maxlen message = self._messages[self._end] del self._message_id_mapping[message.id] + del self._message_metadata[message.id] self._messages[self._end] = None return message @@ -80,6 +85,7 @@ def popleft(self) -> Message: message = self._messages[self._start] del self._message_id_mapping[message.id] + del self._message_metadata[message.id] self._messages[self._start] = None self._start = (self._start + 1) % self.maxlen @@ -89,16 +95,21 @@ def clear(self) -> None: """Remove all messages from the cache.""" self._messages = [None] * self.maxlen self._message_id_mapping = {} + self._message_metadata = {} self._start = 0 self._end = 0 - def get_message(self, message_id: int) -> t.Optional[Message]: + def get_message(self, message_id: int) -> Message | None: """Return the message that has the given message ID, if it is cached.""" index = self._message_id_mapping.get(message_id, None) return self._messages[index] if index is not None else None - def update(self, message: Message) -> bool: + def get_message_metadata(self, message_id: int) -> dict: + """Return the metadata of the message that has the given message ID, if it is cached.""" + return self._message_metadata.get(message_id, None) + + def update(self, message: Message, *, metadata: dict | None = None) -> bool: """ Update a cached message with new contents. @@ -108,13 +119,15 @@ def update(self, message: Message) -> bool: if index is None: return False self._messages[index] = message + if metadata is not None: + self._message_metadata[message.id] = metadata return True def __contains__(self, message_id: int) -> bool: """Return True if the cache contains a message with the given ID .""" return message_id in self._message_id_mapping - def __getitem__(self, item: t.Union[int, slice]) -> t.Union[Message, list[Message]]: + def __getitem__(self, item: int | slice) -> Message | list[Message]: """ Return the message(s) in the index or slice provided. From afd9d4ec993dba3c76ab8a8341b718c75d2a68e6 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 25 Nov 2022 21:00:13 +0200 Subject: [PATCH 072/132] Add nickname filter The nickname filter works in much the same way as the one in the old system, with the following changes: - The lock is per user, rather than a global lock. - The alert cooldown is one hour, instead of three days which seemed too much. The delete_messages setting was changed to the more generic remove_context. If it's a nickname event, the context will be removed by applying a superstar infraction to the user. In order to allow filtering nicknames in voice state events, the filter context can now have None in the channel field. Additionally: - Fixes a bug when ignoring filters in message edits. - Makes the invites list keep track of message edits. - The FakeContext class is moved to utils since it's now also needed by remove_context. --- bot/exts/filtering/_filter_context.py | 3 +- .../filtering/_filter_lists/filter_list.py | 6 +- bot/exts/filtering/_filter_lists/invite.py | 2 +- bot/exts/filtering/_filter_lists/token.py | 2 +- .../filtering/_filters/unique/rich_embed.py | 2 + .../actions/infraction_and_notification.py | 44 ++++---------- .../filtering/_settings_types/actions/ping.py | 2 +- .../{delete_messages.py => remove_context.py} | 57 +++++++++++++++---- .../validations/channel_scope.py | 2 + .../_settings_types/validations/filter_dm.py | 3 + bot/exts/filtering/_ui/ui.py | 22 ++++--- bot/exts/filtering/_utils.py | 35 ++++++++++++ bot/exts/filtering/filtering.py | 56 +++++++++++++++++- 13 files changed, 176 insertions(+), 60 deletions(-) rename bot/exts/filtering/_settings_types/actions/{delete_messages.py => remove_context.py} (52%) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 4a213535ad..61f8c9fbcb 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -18,6 +18,7 @@ class Event(Enum): MESSAGE = auto() MESSAGE_EDIT = auto() + NICKNAME = auto() @dataclass @@ -27,7 +28,7 @@ class FilterContext: # Input context event: Event # The type of event author: User | Member | None # Who triggered the event - channel: TextChannel | Thread | DMChannel # The channel involved + channel: TextChannel | Thread | DMChannel | None # The channel involved content: str | Iterable # What actually needs filtering message: Message | None # The message involved embeds: list = field(default_factory=list) # Any embeds involved diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index b5d6141d75..c829f4a8f8 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -105,10 +105,10 @@ async def _create_filter_list_result( if ctx.event == Event.MESSAGE_EDIT and ctx.message and self.list_type == ListType.DENY: previously_triggered = ctx.message_cache.get_message_metadata(ctx.message.id) + ignore_filters = previously_triggered[self] + # This updates the cache. Some filters are ignored, but they're necessary if there's another edit. + previously_triggered[self] = relevant_filters if previously_triggered and self in previously_triggered: - ignore_filters = previously_triggered[self] - # This updates the cache. Some filters are ignored, but they're necessary if there's another edit. - previously_triggered[self] = relevant_filters relevant_filters = [filter_ for filter_ in relevant_filters if filter_ not in ignore_filters] return relevant_filters diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index 36031f2768..dd14d22223 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -37,7 +37,7 @@ class InviteList(FilterList[InviteFilter]): def __init__(self, filtering_cog: Filtering): super().__init__() - filtering_cog.subscribe(self, Event.MESSAGE) + filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index e4dbf47179..f5da28bb5c 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -32,7 +32,7 @@ class TokensList(FilterList[TokenFilter]): def __init__(self, filtering_cog: Filtering): super().__init__() - filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT, Event.NICKNAME) def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" diff --git a/bot/exts/filtering/_filters/unique/rich_embed.py b/bot/exts/filtering/_filters/unique/rich_embed.py index 5c3517e102..00c28e5710 100644 --- a/bot/exts/filtering/_filters/unique/rich_embed.py +++ b/bot/exts/filtering/_filters/unique/rich_embed.py @@ -21,6 +21,8 @@ async def triggered_on(self, ctx: FilterContext) -> bool: """Determine if `msg` contains any rich embeds not auto-generated from a URL.""" if ctx.embeds: if ctx.event == Event.MESSAGE_EDIT: + if not ctx.message.edited_at: # This might happen, apparently. + return False # If the edit delta is less than 100 microseconds, it's probably a double filter trigger. delta = ctx.message.edited_at - (ctx.before_message.edited_at or ctx.before_message.created_at) if delta.total_seconds() < 0.0001: diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index b8b4636268..f29aee5716 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from datetime import timedelta from enum import Enum, auto from typing import ClassVar @@ -11,45 +10,14 @@ from pydantic import validator import bot as bot_module -from bot.constants import Channels, Guild +from bot.constants import Channels from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +from bot.exts.filtering._utils import FakeContext log = get_logger(__name__) -@dataclass -class FakeContext: - """ - A class representing a context-like object that can be sent to infraction commands. - - The goal is to be able to apply infractions without depending on the existence of a message or an interaction - (which are the two ways to create a Context), e.g. in API events which aren't message-driven, or in custom filtering - events. - """ - - channel: discord.abc.Messageable - bot: bot_module.bot.Bot | None = None - guild: discord.Guild | None = None - author: discord.Member | discord.User | None = None - me: discord.Member | None = None - - def __post_init__(self): - """Initialize the missing information.""" - if not self.bot: - self.bot = bot_module.instance - if not self.guild: - self.guild = self.bot.get_guild(Guild.id) - if not self.me: - self.me = self.guild.me - if not self.author: - self.author = self.me - - async def send(self, *args, **kwargs) -> discord.Message: - """A wrapper for channel.send.""" - return await self.channel.send(*args, **kwargs) - - class Infraction(Enum): """An enumeration of infraction types. The lower the value, the higher it is on the hierarchy.""" @@ -79,6 +47,8 @@ async def invoke( command = bot_module.instance.get_command(command_name) if not command: await alerts_channel.send(f":warning: Could not apply {command_name} to {user.mention}: command not found.") + log.warning(f":warning: Could not apply {command_name} to {user.mention}: command not found.") + return ctx = FakeContext(channel) if self.name in ("KICK", "WARNING", "WATCH", "NOTE"): @@ -160,8 +130,14 @@ async def action(self, ctx: FilterContext) -> None: if not channel: log.info(f"Could not find a channel with ID {self.infraction_channel}, infracting in mod-alerts.") channel = alerts_channel + elif not ctx.channel: + channel = alerts_channel else: channel = ctx.channel + if not channel: # If somehow it's set to `alerts_channel` and it can't be found. + log.error(f"Unable to apply infraction as the context channel {channel} can't be found.") + return + await self.infraction_type.invoke( ctx.author, channel, alerts_channel, self.infraction_duration, self.infraction_reason ) diff --git a/bot/exts/filtering/_settings_types/actions/ping.py b/bot/exts/filtering/_settings_types/actions/ping.py index 5597bdd599..b3725917ca 100644 --- a/bot/exts/filtering/_settings_types/actions/ping.py +++ b/bot/exts/filtering/_settings_types/actions/ping.py @@ -35,7 +35,7 @@ def init_sequence_if_none(cls, pings: list[str] | None) -> list[str]: async def action(self, ctx: FilterContext) -> None: """Add the stored pings to the alert message content.""" - mentions = self.guild_pings if ctx.channel.guild else self.dm_pings + mentions = self.guild_pings if not ctx.channel or ctx.channel.guild else self.dm_pings new_content = " ".join([resolve_mention(mention) for mention in mentions]) ctx.alert_content = f"{new_content} {ctx.alert_content}" diff --git a/bot/exts/filtering/_settings_types/actions/delete_messages.py b/bot/exts/filtering/_settings_types/actions/remove_context.py similarity index 52% rename from bot/exts/filtering/_settings_types/actions/delete_messages.py rename to bot/exts/filtering/_settings_types/actions/remove_context.py index 19c0beb954..7eb3db6c4e 100644 --- a/bot/exts/filtering/_settings_types/actions/delete_messages.py +++ b/bot/exts/filtering/_settings_types/actions/remove_context.py @@ -2,14 +2,24 @@ from typing import ClassVar from botcore.utils import scheduling +from botcore.utils.logging import get_logger from discord import Message from discord.errors import HTTPException +import bot from bot.constants import Channels -from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +from bot.exts.filtering._utils import FakeContext from bot.utils.messages import send_attachments +log = get_logger(__name__) + +SUPERSTAR_REASON = ( + "Your nickname was found to be in violation of our code of conduct. " + "If you believe this is a mistake, please let us know." +) + async def upload_messages_attachments(ctx: FilterContext, messages: list[Message]) -> None: """Re-upload the messages' attachments for future logging.""" @@ -21,22 +31,31 @@ async def upload_messages_attachments(ctx: FilterContext, messages: list[Message ctx.attachments[message.id] = await send_attachments(message, destination, link_large=False) -class DeleteMessages(ActionEntry): +class RemoveContext(ActionEntry): """A setting entry which tells whether to delete the offending message(s).""" - name: ClassVar[str] = "delete_messages" + name: ClassVar[str] = "remove_context" description: ClassVar[str] = ( - "A boolean field. If True, the filter being triggered will cause the offending message to be deleted." + "A boolean field. If True, the filter being triggered will cause the offending context to be removed. " + "An offending message will be deleted, while an offending nickname will be superstarified." ) - delete_messages: bool + remove_context: bool async def action(self, ctx: FilterContext) -> None: - """Delete the context message(s).""" - if not self.delete_messages or not ctx.message: + """Remove the offending context.""" + if not self.remove_context: return - if not ctx.message.guild: + if ctx.event in (Event.MESSAGE, Event.MESSAGE_EDIT): + await self._handle_messages(ctx) + elif ctx.event == Event.NICKNAME: + await self._handle_nickname(ctx) + + @staticmethod + async def _handle_messages(ctx: FilterContext) -> None: + """Delete any messages involved in this context.""" + if not ctx.message or not ctx.message.guild: return channel_messages = defaultdict(set) # Duplicates will cause batch deletion to fail. @@ -68,9 +87,27 @@ async def action(self, ctx: FilterContext) -> None: else: ctx.action_descriptions.append(f"{success} deleted, {fail} failed to delete") + @staticmethod + async def _handle_nickname(ctx: FilterContext) -> None: + """Apply a superstar infraction to remove the user's nickname.""" + alerts_channel = bot.instance.get_channel(Channels.mod_alerts) + if not alerts_channel: + log.error(f"Unable to apply superstar as the context channel {alerts_channel} can't be found.") + return + command = bot.instance.get_command("superstar") + if not command: + user = ctx.author + await alerts_channel.send(f":warning: Could not apply superstar to {user.mention}: command not found.") + log.warning(f":warning: Could not apply superstar to {user.mention}: command not found.") + ctx.action_descriptions.append("failed to superstar") + return + + await command(FakeContext(alerts_channel), ctx.author, None, reason=SUPERSTAR_REASON) + ctx.action_descriptions.append("superstar") + def __or__(self, other: ActionEntry): """Combines two actions of the same type. Each type of action is executed once per filter.""" - if not isinstance(other, DeleteMessages): + if not isinstance(other, RemoveContext): return NotImplemented - return DeleteMessages(delete_messages=self.delete_messages or other.delete_messages) + return RemoveContext(delete_messages=self.remove_context or other.remove_context) diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index 80f837a151..d37efaa099 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -51,6 +51,8 @@ def triggers_on(self, ctx: FilterContext) -> bool: """ channel = ctx.channel + if not channel: + return True if not hasattr(channel, "category"): # This is not a guild channel, outside the scope of this setting. return True if hasattr(channel, "parent"): diff --git a/bot/exts/filtering/_settings_types/validations/filter_dm.py b/bot/exts/filtering/_settings_types/validations/filter_dm.py index b9e566253a..9961984d69 100644 --- a/bot/exts/filtering/_settings_types/validations/filter_dm.py +++ b/bot/exts/filtering/_settings_types/validations/filter_dm.py @@ -14,4 +14,7 @@ class FilterDM(ValidationEntry): def triggers_on(self, ctx: FilterContext) -> bool: """Return whether the filter should be triggered even if it was triggered in DMs.""" + if not ctx.channel: # No channel - out of scope for this setting. + return True + return ctx.channel.guild is not None or self.filter_dm diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 17a9337837..ec549725c0 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -2,6 +2,7 @@ import re from abc import ABC, abstractmethod +from collections.abc import Iterable from enum import EnumMeta from functools import partial from typing import Any, Callable, Coroutine, Optional, TypeVar @@ -72,17 +73,21 @@ async def _build_alert_message_content(ctx: FilterContext, current_message_lengt return alert_content -async def build_mod_alert(ctx: FilterContext, triggered_filters: dict[FilterList, list[str]]) -> Embed: +async def build_mod_alert(ctx: FilterContext, triggered_filters: dict[FilterList, Iterable[str]]) -> Embed: """Build an alert message from the filter context.""" embed = Embed(color=Colours.soft_orange) embed.set_thumbnail(url=ctx.author.display_avatar.url) triggered_by = f"**Triggered by:** {format_user(ctx.author)}" - if ctx.channel.guild: - triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}\n" + if ctx.channel: + if ctx.channel.guild: + triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}\n" + else: + triggered_in = "**Triggered in:** :warning:**DM**:warning:\n" + if len(ctx.related_channels) > 1: + triggered_in += f"**Channels:** {', '.join(channel.mention for channel in ctx.related_channels)}\n" else: - triggered_in = "**Triggered in:** :warning:**DM**:warning:\n" - if len(ctx.related_channels) > 1: - triggered_in += f"**Channels:** {', '.join(channel.mention for channel in ctx.related_channels)}\n" + triggered_by += "\n" + triggered_in = "" filters = [] for filter_list, list_message in triggered_filters.items(): @@ -94,7 +99,10 @@ async def build_mod_alert(ctx: FilterContext, triggered_filters: dict[FilterList actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") mod_alert_message = "\n".join(part for part in (triggered_by, triggered_in, filters, matches, actions) if part) - mod_alert_message += f"\n**[Original Content]({ctx.message.jump_url})**:\n" + if ctx.message: + mod_alert_message += f"\n**[Original Content]({ctx.message.jump_url})**:\n" + else: + mod_alert_message += "\n**Original Content**:\n" mod_alert_message += await _build_alert_message_content(ctx, len(mod_alert_message)) embed.description = mod_alert_message diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index 86b6ab1011..bd56c12602 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -4,12 +4,15 @@ import pkgutil from abc import ABC, abstractmethod from collections import defaultdict +from dataclasses import dataclass from functools import cache from typing import Any, Iterable, TypeVar, Union +import discord import regex import bot +from bot.bot import Bot from bot.constants import Guild VARIATION_SELECTORS = r"\uFE00-\uFE0F\U000E0100-\U000E01EF" @@ -183,3 +186,35 @@ def inherited(attr: str) -> bool: else: # Add to the set of unique values for that field. FieldRequiring.__unique_attributes[parent][attribute].add(value) + + +@dataclass +class FakeContext: + """ + A class representing a context-like object that can be sent to infraction commands. + + The goal is to be able to apply infractions without depending on the existence of a message or an interaction + (which are the two ways to create a Context), e.g. in API events which aren't message-driven, or in custom filtering + events. + """ + + channel: discord.abc.Messageable + bot: Bot | None = None + guild: discord.Guild | None = None + author: discord.Member | discord.User | None = None + me: discord.Member | None = None + + def __post_init__(self): + """Initialize the missing information.""" + if not self.bot: + self.bot = bot.instance + if not self.guild: + self.guild = self.bot.get_guild(Guild.id) + if not self.me: + self.me = self.guild.me + if not self.author: + self.author = self.me + + async def send(self, *args, **kwargs) -> discord.Message: + """A wrapper for channel.send.""" + return await self.channel.send(*args, **kwargs) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 05b2339b95..9c9b1eff4b 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -2,13 +2,17 @@ import json import operator import re +import unicodedata from collections import defaultdict +from collections.abc import Iterable from functools import partial, reduce from io import BytesIO +from operator import attrgetter from typing import Literal, Optional, get_type_hints import arrow import discord +from async_rediscache import RedisCache from botcore.site_api import ResponseCodeError from discord import Colour, Embed, HTTPException, Message, MessageType from discord.ext import commands, tasks @@ -37,11 +41,13 @@ from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.channel import is_mod_channel +from bot.utils.lock import lock_arg from bot.utils.message_cache import MessageCache log = get_logger(__name__) CACHE_SIZE = 100 +HOURS_BETWEEN_NICKNAME_ALERTS = 1 WEEKLY_REPORT_ISO_DAY = 3 # 1=Monday, 7=Sunday @@ -51,6 +57,9 @@ class Filtering(Cog): # A set of filter list names with missing implementations that already caused a warning. already_warned = set() + # Redis cache mapping a user ID to the last timestamp a bad nickname alert was sent. + name_alerts = RedisCache() + # region: init def __init__(self, bot: Bot): @@ -188,6 +197,10 @@ async def on_message(self, msg: Message) -> None: if ctx.send_alert: await self._send_alert(ctx, list_messages) + ctx = FilterContext.from_message(Event.NICKNAME, msg) + ctx.content = msg.author.display_name + await self._check_bad_name(ctx) + @Cog.listener() async def on_message_edit(self, before: discord.Message, after: discord.Message) -> None: """Filter the contents of an edited message. Don't reinvoke filters already invoked on the `before` version.""" @@ -209,6 +222,12 @@ async def on_message_edit(self, before: discord.Message, after: discord.Message) if ctx.send_alert: await self._send_alert(ctx, list_messages) + @Cog.listener() + async def on_voice_state_update(self, member: discord.Member, *_) -> None: + """Checks for bad words in usernames when users join, switch or leave a voice channel.""" + ctx = FilterContext(Event.NICKNAME, member, None, member.display_name, None) + await self._check_bad_name(ctx) + # endregion # region: blacklist commands @@ -388,7 +407,7 @@ async def f_add( A template filter can be specified in the settings area to copy overrides from. The setting name is "--template" and the value is the filter ID. The template will be used before applying any other override. - Example: `!filter add denied token "Scaleios is great" delete_messages=True send_alert=False --template=100` + Example: `!filter add denied token "Scaleios is great" remove_context=True send_alert=False --template=100` """ result = await self._resolve_list_type_and_name(ctx, list_type, list_name) if result is None: @@ -809,7 +828,7 @@ async def _resolve_action( return result_actions, messages, triggers - async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, list[str]]) -> None: + async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterList, Iterable[str]]) -> None: """Build an alert message from the filter context, and send it via the alert webhook.""" if not self.webhook: return @@ -819,6 +838,39 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi # There shouldn't be more than 10, but if there are it's not very useful to send them all. await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10]) + async def _recently_alerted_name(self, member: discord.Member) -> bool: + """When it hasn't been `HOURS_BETWEEN_NICKNAME_ALERTS` since last alert, return False, otherwise True.""" + if last_alert := await self.name_alerts.get(member.id): + last_alert = arrow.get(last_alert) + if arrow.utcnow() - last_alert < datetime.timedelta(days=HOURS_BETWEEN_NICKNAME_ALERTS): + log.trace(f"Last alert was too recent for {member}'s nickname.") + return True + + return False + + @lock_arg("filtering.check_bad_name", "ctx", attrgetter("author.id")) + async def _check_bad_name(self, ctx: FilterContext) -> None: + """Check filter triggers in the passed context - a member's display name.""" + if await self._recently_alerted_name(ctx.author): + return + + name = ctx.content + normalised_name = unicodedata.normalize("NFKC", name) + cleaned_normalised_name = "".join([c for c in normalised_name if not unicodedata.combining(c)]) + + # Run filters against normalised, cleaned normalised and the original name, + # in case there are filters for one but not another. + names_to_check = (name, normalised_name, cleaned_normalised_name) + + new_ctx = ctx.replace(content=" ".join(names_to_check)) + result_actions, list_messages, _ = await self._resolve_action(new_ctx) + if result_actions: + await result_actions.action(ctx) + if ctx.send_alert: + await self._send_alert(ctx, list_messages) # `ctx` has the original content. + # Update time when alert sent + await self.name_alerts.set(ctx.author.id, arrow.utcnow().timestamp()) + async def _resolve_list_type_and_name( self, ctx: Context, list_type: ListType | None = None, list_name: str | None = None, *, exclude: str = "" ) -> tuple[ListType, FilterList] | None: From 7591c5e58d97d264505302c984acca32da143fc7 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 25 Nov 2022 23:43:03 +0200 Subject: [PATCH 073/132] Add offending message deletion scheduling This is mostly a copy-paste from the old system. If there are any actions taken against the message, but deleting is not one of them - schedule the message for deletion after 7 days. --- bot/exts/filtering/filtering.py | 77 ++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 9c9b1eff4b..65089a6d55 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -4,7 +4,7 @@ import re import unicodedata from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from functools import partial, reduce from io import BytesIO from operator import attrgetter @@ -14,6 +14,7 @@ import discord from async_rediscache import RedisCache from botcore.site_api import ResponseCodeError +from botcore.utils import scheduling from discord import Colour, Embed, HTTPException, Message, MessageType from discord.ext import commands, tasks from discord.ext.commands import BadArgument, Cog, Context, command, has_any_role @@ -48,6 +49,7 @@ CACHE_SIZE = 100 HOURS_BETWEEN_NICKNAME_ALERTS = 1 +OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=7) WEEKLY_REPORT_ISO_DAY = 3 # 1=Monday, 7=Sunday @@ -66,6 +68,7 @@ def __init__(self, bot: Bot): self.bot = bot self.filter_lists: dict[str, FilterList] = {} self._subscriptions: defaultdict[Event, list[FilterList]] = defaultdict(list) + self.delete_scheduler = scheduling.Scheduler(self.__class__.__name__) self.webhook = None self.loaded_settings = {} @@ -95,6 +98,7 @@ async def cog_load(self) -> None: log.error(f"Failed to fetch filters webhook with ID `{Webhooks.filters}`.") self.collect_loaded_types(example_list) + await self.schedule_offending_messages_deletion() self.weekly_auto_infraction_report_task.start() def subscribe(self, filter_list: FilterList, *events: Event) -> None: @@ -175,6 +179,18 @@ def collect_loaded_types(self, example_list: AtomicList) -> None: for field_name in extra_fields_type.__fields__ } + async def schedule_offending_messages_deletion(self) -> None: + """Load the messages that need to be scheduled for deletion from the database.""" + response = await self.bot.api_client.get('bot/offensive-messages') + + now = arrow.utcnow() + for msg in response: + delete_at = arrow.get(msg['delete_date']) + if delete_at < now: + await self._delete_offensive_msg(msg) + else: + self._schedule_msg_delete(msg) + async def cog_check(self, ctx: Context) -> bool: """Only allow moderators to invoke the commands in this cog.""" return await has_any_role(*MODERATION_ROLES).predicate(ctx) @@ -197,9 +213,11 @@ async def on_message(self, msg: Message) -> None: if ctx.send_alert: await self._send_alert(ctx, list_messages) - ctx = FilterContext.from_message(Event.NICKNAME, msg) - ctx.content = msg.author.display_name - await self._check_bad_name(ctx) + nick_ctx = FilterContext.from_message(Event.NICKNAME, msg) + nick_ctx.content = msg.author.display_name + await self._check_bad_name(nick_ctx) + + await self._maybe_schedule_msg_delete(ctx, result_actions) @Cog.listener() async def on_message_edit(self, before: discord.Message, after: discord.Message) -> None: @@ -221,6 +239,7 @@ async def on_message_edit(self, before: discord.Message, after: discord.Message) await result_actions.action(ctx) if ctx.send_alert: await self._send_alert(ctx, list_messages) + await self._maybe_schedule_msg_delete(ctx, result_actions) @Cog.listener() async def on_voice_state_update(self, member: discord.Member, *_) -> None: @@ -1187,6 +1206,53 @@ async def _search_filters( ctx = await bot.instance.get_context(message) await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False, reply=True) + async def _delete_offensive_msg(self, msg: Mapping[str, int]) -> None: + """Delete an offensive message, and then delete it from the DB.""" + try: + channel = self.bot.get_channel(msg['channel_id']) + if channel: + msg_obj = await channel.fetch_message(msg['id']) + await msg_obj.delete() + except discord.NotFound: + log.info( + f"Tried to delete message {msg['id']}, but the message can't be found " + f"(it has been probably already deleted)." + ) + except HTTPException as e: + log.warning(f"Failed to delete message {msg['id']}: status {e.status}") + + await self.bot.api_client.delete(f'bot/offensive-messages/{msg["id"]}') + log.info(f"Deleted the offensive message with id {msg['id']}.") + + def _schedule_msg_delete(self, msg: dict) -> None: + """Delete an offensive message once its deletion date is reached.""" + delete_at = arrow.get(msg['delete_date']).datetime + self.delete_scheduler.schedule_at(delete_at, msg['id'], self._delete_offensive_msg(msg)) + + async def _maybe_schedule_msg_delete(self, ctx: FilterContext, actions: ActionSettings | None) -> None: + """Post the message to the database and schedule it for deletion if it's not set to be deleted already.""" + msg = ctx.message + if not msg or not actions or actions.get_setting("remove_context", True): + return + + delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() + data = { + 'id': msg.id, + 'channel_id': msg.channel.id, + 'delete_date': delete_date + } + + try: + await self.bot.api_client.post('bot/offensive-messages', json=data) + except ResponseCodeError as e: + if e.status == 400 and "already exists" in e.response_json.get("id", [""])[0]: + log.debug(f"Offensive message {msg.id} already exists.") + else: + log.error(f"Offensive message {msg.id} failed to post: {e}") + else: + self._schedule_msg_delete(data) + log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") + # endregion # region: tasks @@ -1242,8 +1308,9 @@ async def send_weekly_auto_infraction_report(self, channel: discord.TextChannel # endregion async def cog_unload(self) -> None: - """Cancel the weekly auto-infraction filter report on cog unload.""" + """Cancel the weekly auto-infraction filter report and deletion scheduling on cog unload.""" self.weekly_auto_infraction_report_task.cancel() + self.delete_scheduler.cancel_all() async def setup(bot: Bot) -> None: From b950f64ed5cba6dff5c2b97e3b37f7409670d031 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 26 Nov 2022 12:59:49 +0200 Subject: [PATCH 074/132] Bring back snekbox result filtering With the new system, some custom actions might be applied if any filters are triggered, for example message deletion or infraction. Those actions will be applied in the context of the command invocation. --- bot/exts/filtering/_filter_lists/antispam.py | 2 +- bot/exts/filtering/filtering.py | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index cf58757238..41eda9878e 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -59,7 +59,7 @@ async def actions_for( self, ctx: FilterContext ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" - if not ctx.message: + if not ctx.message or not ctx.message_cache: return None, [], {} sublist: SubscribingAtomicList = self[ListType.DENY] diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 65089a6d55..81348ca1ed 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -196,7 +196,7 @@ async def cog_check(self, ctx: Context) -> bool: return await has_any_role(*MODERATION_ROLES).predicate(ctx) # endregion - # region: listeners + # region: listeners and event handlers @Cog.listener() async def on_message(self, msg: Message) -> None: @@ -247,6 +247,24 @@ async def on_voice_state_update(self, member: discord.Member, *_) -> None: ctx = FilterContext(Event.NICKNAME, member, None, member.display_name, None) await self._check_bad_name(ctx) + async def filter_snekbox_output(self, snekbox_result: str, msg: Message) -> bool: + """ + Filter the result of a snekbox command to see if it violates any of our rules, and then respond accordingly. + + Also requires the original message, to check whether to filter and for alerting. + Any action (deletion, infraction) will be applied in the context of the original message. + + Returns whether a filter was triggered or not. + """ + ctx = FilterContext.from_message(Event.MESSAGE, msg).replace(content=snekbox_result) + result_actions, list_messages, triggers = await self._resolve_action(ctx) + if result_actions: + await result_actions.action(ctx) + if ctx.send_alert: + await self._send_alert(ctx, list_messages) + + return result_actions is not None + # endregion # region: blacklist commands From 2fab8c1962666934d42cea1aa7b7bb79fe3b5a09 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 26 Nov 2022 15:53:42 +0200 Subject: [PATCH 075/132] Phishing filter add command A shortcut command to autoban a compromised account if it posts a given content. --- .../actions/infraction_and_notification.py | 2 +- .../_settings_types/actions/remove_context.py | 2 +- bot/exts/filtering/_utils.py | 2 ++ bot/exts/filtering/filtering.py | 22 +++++++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index f29aee5716..ed08362de5 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -50,7 +50,7 @@ async def invoke( log.warning(f":warning: Could not apply {command_name} to {user.mention}: command not found.") return - ctx = FakeContext(channel) + ctx = FakeContext(channel, command) if self.name in ("KICK", "WARNING", "WATCH", "NOTE"): await command(ctx, user, reason=reason or None) else: diff --git a/bot/exts/filtering/_settings_types/actions/remove_context.py b/bot/exts/filtering/_settings_types/actions/remove_context.py index 7eb3db6c4e..2ba7313091 100644 --- a/bot/exts/filtering/_settings_types/actions/remove_context.py +++ b/bot/exts/filtering/_settings_types/actions/remove_context.py @@ -102,7 +102,7 @@ async def _handle_nickname(ctx: FilterContext) -> None: ctx.action_descriptions.append("failed to superstar") return - await command(FakeContext(alerts_channel), ctx.author, None, reason=SUPERSTAR_REASON) + await command(FakeContext(alerts_channel, command), ctx.author, None, reason=SUPERSTAR_REASON) ctx.action_descriptions.append("superstar") def __or__(self, other: ActionEntry): diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index bd56c12602..5abb2240b8 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -10,6 +10,7 @@ import discord import regex +from discord.ext.commands import Command import bot from bot.bot import Bot @@ -199,6 +200,7 @@ class FakeContext: """ channel: discord.abc.Messageable + command: Command | None bot: Bot | None = None guild: discord.Guild | None = None author: discord.Member | discord.User | None = None diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 81348ca1ed..673b5487c5 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -39,6 +39,7 @@ ArgumentCompletionView, DeleteConfirmationView, build_mod_alert, format_response_error ) from bot.exts.filtering._utils import past_tense, repr_equals, starting_value, to_serializable +from bot.exts.moderation.infraction.infractions import COMP_BAN_DURATION, COMP_BAN_REASON from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.channel import is_mod_channel @@ -671,6 +672,27 @@ async def f_search( ) await ctx.send(embed=embed, reference=ctx.message, view=view) + @filter.command(root_aliases=("compfilter", "compf")) + async def compadd( + self, ctx: Context, list_name: Optional[str], content: str, *, description: Optional[str] = "Phishing" + ) -> None: + """Add a filter to detect a compromised account. Will apply the equivalent of a compban if triggered.""" + result = await self._resolve_list_type_and_name(ctx, ListType.DENY, list_name, exclude="list_type") + if result is None: + return + list_type, filter_list = result + + settings = ( + "remove_context=True " + "dm_pings=Moderators " + "infraction_type=BAN " + "infraction_channel=1 " # Post the ban in #mod-alerts + f"infraction_duration={COMP_BAN_DURATION.total_seconds()} " + f"infraction_reason={COMP_BAN_REASON}" + ) + description_and_settings = f"{description} {settings}" + await self._add_filter(ctx, "noui", list_type, filter_list, content, description_and_settings) + # endregion # region: filterlist group From 2def5b6b4da89ae1579f2f16df2a70a69953bf42 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 26 Nov 2022 21:27:17 +0200 Subject: [PATCH 076/132] Add alert view The view provides info about the offending user. This change required changing the cog to generate a fresh webhook on startup so that it's authenticated to the bot. Only webhooks authenticated to the bot can send message components. In the future, this view might be used for more advanced usages such as having a button to pardon the user. --- bot/constants.py | 1 - bot/exts/filtering/_filter_lists/antispam.py | 14 +++---- bot/exts/filtering/_ui/ui.py | 40 ++++++++++++++++++++ bot/exts/filtering/filtering.py | 36 ++++++++++++++---- config-default.yml | 1 - 5 files changed, 74 insertions(+), 18 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 8a2571a985..1d6ab8e7e0 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -455,7 +455,6 @@ class Webhooks(metaclass=YAMLGetter): duck_pond: int incidents: int incidents_archive: int - filters: int class Roles(metaclass=YAMLGetter): diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index 41eda9878e..e549404c47 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -10,17 +10,15 @@ import arrow from botcore.utils import scheduling from botcore.utils.logging import get_logger -from discord import HTTPException, Member +from discord import Member -import bot -from bot.constants import Webhooks from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filter_lists.filter_list import ListType, SubscribingAtomicList, UniquesListBase from bot.exts.filtering._filters.antispam import antispam_filter_types from bot.exts.filtering._filters.filter import Filter, UniqueFilter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction -from bot.exts.filtering._ui.ui import build_mod_alert +from bot.exts.filtering._ui.ui import AlertView, build_mod_alert if typing.TYPE_CHECKING: from bot.exts.filtering.filtering import Filtering @@ -147,9 +145,9 @@ async def send_alert(self, antispam_list: AntispamList) -> None: """Post the mod alert.""" if not self.contexts or not self.rules: return - try: - webhook = await bot.instance.fetch_webhook(Webhooks.filters) - except HTTPException: + + webhook = antispam_list.filtering_cog.webhook + if not webhook: return ctx, *other_contexts = self.contexts @@ -182,4 +180,4 @@ async def send_alert(self, antispam_list: AntispamList) -> None: embed.set_footer( text="The list of actions taken includes actions from additional contexts after deletion began." ) - await webhook.send(username="Anti-Spam", content=ctx.alert_content, embeds=[embed]) + await webhook.send(username="Anti-Spam", content=ctx.alert_content, embeds=[embed], view=AlertView(new_ctx)) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index ec549725c0..e71bab0d16 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -20,6 +20,7 @@ from bot.constants import Colours from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filter_lists import FilterList +from bot.exts.filtering._utils import FakeContext from bot.utils.messages import format_channel, format_user, upload_log log = get_logger(__name__) @@ -40,6 +41,8 @@ # Max number of items in a select MAX_SELECT_ITEMS = 25 MAX_EMBED_DESCRIPTION = 4080 +# Number of seconds before timeout of the alert view +ALERT_VIEW_TIMEOUT = 3600 SETTINGS_DELIMITER = re.compile(r"\s+(?=\S+=\S+)") SINGLE_SETTING_PATTERN = re.compile(r"[\w/]+=.+") @@ -502,3 +505,40 @@ async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> async def cancel(self, interaction: Interaction, button: discord.ui.Button) -> None: """Cancel the filter list deletion.""" await interaction.response.edit_message(content="🚫 Operation canceled.", view=None) + + +class AlertView(discord.ui.View): + """A view providing info about the offending user.""" + + def __init__(self, ctx: FilterContext): + super().__init__(timeout=ALERT_VIEW_TIMEOUT) + self.ctx = ctx + + @discord.ui.button(label="ID") + async def user_id(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Reply with the ID of the offending user.""" + await interaction.response.send_message(self.ctx.author.id, ephemeral=True) + + @discord.ui.button(emoji="👤") + async def user_info(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Send the info embed of the offending user.""" + command = bot.instance.get_command("user") + if not command: + await interaction.response.send_message("The command `user` is not loaded.", ephemeral=True) + return + + await interaction.response.defer() + fake_ctx = FakeContext(interaction.channel, command, author=interaction.user) + await command(fake_ctx, self.ctx.author) + + @discord.ui.button(emoji="⚠") + async def user_infractions(self, interaction: Interaction, button: discord.ui.Button) -> None: + """Send the infractions embed of the offending user.""" + command = bot.instance.get_command("infraction search") + if not command: + await interaction.response.send_message("The command `infraction search` is not loaded.", ephemeral=True) + return + + await interaction.response.defer() + fake_ctx = FakeContext(interaction.channel, command, author=interaction.user) + await command(fake_ctx, self.ctx.author) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 673b5487c5..2e433aff65 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -23,7 +23,8 @@ import bot.exts.filtering._ui.filter as filters_ui from bot import constants from bot.bot import Bot -from bot.constants import Channels, MODERATION_ROLES, Roles, Webhooks +from bot.constants import Channels, Guild, MODERATION_ROLES, Roles +from bot.exts.backend.branding._repository import HEADERS, PARAMS from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filter_lists.filter_list import AtomicList @@ -36,7 +37,7 @@ from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter from bot.exts.filtering._ui.search import SearchEditView, search_criteria_converter from bot.exts.filtering._ui.ui import ( - ArgumentCompletionView, DeleteConfirmationView, build_mod_alert, format_response_error + AlertView, ArgumentCompletionView, DeleteConfirmationView, build_mod_alert, format_response_error ) from bot.exts.filtering._utils import past_tense, repr_equals, starting_value, to_serializable from bot.exts.moderation.infraction.infractions import COMP_BAN_DURATION, COMP_BAN_REASON @@ -48,6 +49,7 @@ log = get_logger(__name__) +WEBHOOK_ICON_URL = r"https://github.com/python-discord/branding/raw/main/icons/filter/filter_pfp.png" CACHE_SIZE = 100 HOURS_BETWEEN_NICKNAME_ALERTS = 1 OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=7) @@ -70,7 +72,7 @@ def __init__(self, bot: Bot): self.filter_lists: dict[str, FilterList] = {} self._subscriptions: defaultdict[Event, list[FilterList]] = defaultdict(list) self.delete_scheduler = scheduling.Scheduler(self.__class__.__name__) - self.webhook = None + self.webhook: discord.Webhook = None self.loaded_settings = {} self.loaded_filters = {} @@ -93,10 +95,8 @@ async def cog_load(self) -> None: if not example_list and loaded_list: example_list = loaded_list - try: - self.webhook = await self.bot.fetch_webhook(Webhooks.filters) - except HTTPException: - log.error(f"Failed to fetch filters webhook with ID `{Webhooks.filters}`.") + # The webhook must be generated by the bot to send messages with components through it. + self.webhook = await self._generate_webhook() self.collect_loaded_types(example_list) await self.schedule_offending_messages_deletion() @@ -861,6 +861,24 @@ def _load_raw_filter_list(self, list_data: dict) -> AtomicList | None: self.filter_lists[list_name] = filter_list_types[list_name](self) return self.filter_lists[list_name].add_list(list_data) + async def _generate_webhook(self) -> discord.Webhook | None: + """Generate a webhook with the filtering avatar.""" + # Download the filtering avatar from the branding repository. + webhook_icon = None + async with self.bot.http_session.get(WEBHOOK_ICON_URL, params=PARAMS, headers=HEADERS) as response: + if response.status == 200: + log.debug("Successfully fetched filtering webhook icon, reading payload.") + webhook_icon = await response.read() + else: + log.warning(f"Failed to fetch filtering webhook icon due to status: {response.status}") + + alerts_channel = self.bot.get_guild(Guild.id).get_channel(Channels.mod_alerts) + try: + return await alerts_channel.create_webhook(name="Filtering System", avatar=webhook_icon) + except HTTPException: + log.error("Failed to create filters webhook.") + return None + async def _resolve_action( self, ctx: FilterContext ) -> tuple[ActionSettings | None, dict[FilterList, list[str]], dict[AtomicList, list[Filter]]]: @@ -895,7 +913,9 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi name = f"{ctx.event.name.replace('_', ' ').title()} Filter" embed = await build_mod_alert(ctx, triggered_filters) # There shouldn't be more than 10, but if there are it's not very useful to send them all. - await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10]) + await self.webhook.send( + username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10], view=AlertView(ctx) + ) async def _recently_alerted_name(self, member: discord.Member) -> bool: """When it hasn't been `HOURS_BETWEEN_NICKNAME_ALERTS` since last alert, return False, otherwise True.""" diff --git a/config-default.yml b/config-default.yml index 4407177d97..f0e217d6cf 100644 --- a/config-default.yml +++ b/config-default.yml @@ -324,7 +324,6 @@ guild: incidents: 816650601844572212 incidents_archive: 720671599790915702 python_news: &PYNEWS_WEBHOOK 704381182279942324 - filters: 926442964463521843 keys: From c3b2e55bea5933d005baac9ac9e35a4c214a8473 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 27 Nov 2022 19:50:44 +0200 Subject: [PATCH 077/132] fix: Validate filter list before initiating filter add --- bot/exts/filtering/filtering.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 2e433aff65..4f4cda611b 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1020,10 +1020,15 @@ async def _add_filter( description_and_settings: Optional[str] = None ) -> None: """Add a filter to the database.""" + # Validations. + if list_type not in filter_list: + await ctx.reply(f":x: There is no list of {past_tense(list_type.name.lower())} {filter_list.name}s.") + return filter_type = filter_list.get_filter_type(content) if not filter_type: await ctx.reply(f":x: Could not find a filter type appropriate for `{content}`.") return + # Parse the description and settings. description, settings, filter_settings = description_and_settings_converter( filter_list, list_type, @@ -1033,7 +1038,7 @@ async def _add_filter( description_and_settings ) - if noui: + if noui: # Add directly with no UI. try: await self._post_new_filter( ctx.message, filter_list, list_type, filter_type, content, description, settings, filter_settings @@ -1043,7 +1048,7 @@ async def _add_filter( except ValueError as e: raise BadArgument(str(e)) return - + # Bring up the UI. embed = Embed(colour=Colour.blue()) embed.description = f"`{content}`" if content else "*No content*" if description: From 8f2f188cee48096aebf5b7aeff04d632643dc7be Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 27 Nov 2022 20:14:03 +0200 Subject: [PATCH 078/132] fix: Handle uncached messages in filter editing This also increases the cache size to increase the chances of ignoring filters. Since it's newest first and the antispam uses take_while, it doesn't slow down the antispam. --- bot/exts/filtering/_filter_lists/filter_list.py | 9 +++++---- bot/exts/filtering/filtering.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index c829f4a8f8..64c96d3504 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -105,10 +105,11 @@ async def _create_filter_list_result( if ctx.event == Event.MESSAGE_EDIT and ctx.message and self.list_type == ListType.DENY: previously_triggered = ctx.message_cache.get_message_metadata(ctx.message.id) - ignore_filters = previously_triggered[self] - # This updates the cache. Some filters are ignored, but they're necessary if there's another edit. - previously_triggered[self] = relevant_filters - if previously_triggered and self in previously_triggered: + # The message might not be cached. + if previously_triggered: + ignore_filters = previously_triggered[self] + # This updates the cache. Some filters are ignored, but they're necessary if there's another edit. + previously_triggered[self] = relevant_filters relevant_filters = [filter_ for filter_ in relevant_filters if filter_ not in ignore_filters] return relevant_filters diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 4f4cda611b..3fb627a821 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -50,7 +50,7 @@ log = get_logger(__name__) WEBHOOK_ICON_URL = r"https://github.com/python-discord/branding/raw/main/icons/filter/filter_pfp.png" -CACHE_SIZE = 100 +CACHE_SIZE = 1000 HOURS_BETWEEN_NICKNAME_ALERTS = 1 OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=7) WEEKLY_REPORT_ISO_DAY = 3 # 1=Monday, 7=Sunday From 1e5667e15dd5f4ec2633d08fc4262e410c2f1ed4 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 27 Nov 2022 20:42:17 +0200 Subject: [PATCH 079/132] fix: Use existing filters webhook if found --- bot/exts/filtering/filtering.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 3fb627a821..bf38a2818c 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -50,6 +50,7 @@ log = get_logger(__name__) WEBHOOK_ICON_URL = r"https://github.com/python-discord/branding/raw/main/icons/filter/filter_pfp.png" +WEBHOOK_NAME = "Filtering System" CACHE_SIZE = 1000 HOURS_BETWEEN_NICKNAME_ALERTS = 1 OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=7) @@ -96,7 +97,7 @@ async def cog_load(self) -> None: example_list = loaded_list # The webhook must be generated by the bot to send messages with components through it. - self.webhook = await self._generate_webhook() + self.webhook = await self._fetch_or_generate_filtering_webhook() self.collect_loaded_types(example_list) await self.schedule_offending_messages_deletion() @@ -861,8 +862,15 @@ def _load_raw_filter_list(self, list_data: dict) -> AtomicList | None: self.filter_lists[list_name] = filter_list_types[list_name](self) return self.filter_lists[list_name].add_list(list_data) - async def _generate_webhook(self) -> discord.Webhook | None: + async def _fetch_or_generate_filtering_webhook(self) -> discord.Webhook | None: """Generate a webhook with the filtering avatar.""" + alerts_channel = self.bot.get_guild(Guild.id).get_channel(Channels.mod_alerts) + # Try to find an existing webhook. + for webhook in await alerts_channel.webhooks(): + if webhook.name == WEBHOOK_NAME and webhook.user == self.bot.user and webhook.is_authenticated(): + log.trace(f"Found existing filters webhook with ID {webhook.id}.") + return webhook + # Download the filtering avatar from the branding repository. webhook_icon = None async with self.bot.http_session.get(WEBHOOK_ICON_URL, params=PARAMS, headers=HEADERS) as response: @@ -872,11 +880,13 @@ async def _generate_webhook(self) -> discord.Webhook | None: else: log.warning(f"Failed to fetch filtering webhook icon due to status: {response.status}") - alerts_channel = self.bot.get_guild(Guild.id).get_channel(Channels.mod_alerts) + # Generate a new webhook. try: - return await alerts_channel.create_webhook(name="Filtering System", avatar=webhook_icon) - except HTTPException: - log.error("Failed to create filters webhook.") + webhook = await alerts_channel.create_webhook(name=WEBHOOK_NAME, avatar=webhook_icon) + log.trace(f"Generated new filters webhook with ID {webhook.id},") + return webhook + except HTTPException as e: + log.error(f"Failed to create filters webhook: {e}") return None async def _resolve_action( From afab00965fb760c414504463a71d327df8706cd6 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 28 Nov 2022 03:18:11 +0200 Subject: [PATCH 080/132] fix setting name --- bot/exts/filtering/_settings_types/actions/remove_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_settings_types/actions/remove_context.py b/bot/exts/filtering/_settings_types/actions/remove_context.py index 2ba7313091..4107400282 100644 --- a/bot/exts/filtering/_settings_types/actions/remove_context.py +++ b/bot/exts/filtering/_settings_types/actions/remove_context.py @@ -110,4 +110,4 @@ def __or__(self, other: ActionEntry): if not isinstance(other, RemoveContext): return NotImplemented - return RemoveContext(delete_messages=self.remove_context or other.remove_context) + return RemoveContext(remove_context=self.remove_context or other.remove_context) From 71bb23082e4a3a48368e9046d4ddbf50518641f6 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 28 Nov 2022 22:17:22 +0200 Subject: [PATCH 081/132] fix: Remove hash prefix from filter IDs --- bot/exts/filtering/_filter_lists/filter_list.py | 2 +- bot/exts/filtering/_filters/filter.py | 2 +- bot/exts/filtering/filtering.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 64c96d3504..a32b0741e2 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -149,7 +149,7 @@ def format_messages(triggers: list[Filter], *, expand_single_filter: bool = True message += f" - {triggers[0].description}" messages = [message] else: - messages = [f"#{filter_.id} (`{filter_.content}`)" for filter_ in triggers] + messages = [f"{filter_.id} (`{filter_.content}`)" for filter_ in triggers] return messages def __hash__(self): diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 49c163d993..b5f4c127a1 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -78,7 +78,7 @@ async def process_input(cls, content: str, description: str) -> tuple[str, str]: def __str__(self) -> str: """A string representation of the filter.""" - string = f"#{self.id}. `{self.content}`" + string = f"{self.id}. `{self.content}`" if self.description: string += f" - {self.description}" return string diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index bf38a2818c..8b1a48617d 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -383,7 +383,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: embed.description = f"`{filter_.content}`" if filter_.description: embed.description += f" - {filter_.description}" - embed.set_author(name=f"Filter #{id_} - " + f"{filter_list[list_type].label}".title()) + embed.set_author(name=f"Filter {id_} - " + f"{filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " f"To view all defaults of the list, " @@ -512,7 +512,7 @@ async def f_edit( if description: embed.description += f" - {description}" embed.set_author( - name=f"Filter #{filter_id} - {filter_list[list_type].label}".title()) + name=f"Filter {filter_id} - {filter_list[list_type].label}".title()) embed.set_footer(text=( "Field names with an asterisk have values which override the defaults of the containing filter list. " f"To view all defaults of the list, " From 58949ce36c321d6ffd528e346b974820b90b858b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 28 Nov 2022 22:51:07 +0200 Subject: [PATCH 082/132] fix: Correctly handle author being a User If an offending message is sent in DMs, infractions that require the user being on the server didn't work, and there was an error while trying to display the user embed from that object, since the user is on the server (and therefore should have a nick attribute). --- .../_settings_types/actions/infraction_and_notification.py | 7 ++++++- bot/exts/filtering/_ui/ui.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index ed08362de5..d60bcac5ba 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -5,6 +5,7 @@ import arrow import discord.abc from botcore.utils.logging import get_logger +from botcore.utils.members import get_or_fetch_member from discord import Colour, Embed, Member, User from discord.errors import Forbidden from pydantic import validator @@ -37,7 +38,7 @@ def __str__(self) -> str: async def invoke( self, user: Member | User, - channel: discord.abc.Messageable, + channel: discord.abc.GuildChannel | discord.DMChannel, alerts_channel: discord.TextChannel, duration: float, reason: str @@ -50,6 +51,10 @@ async def invoke( log.warning(f":warning: Could not apply {command_name} to {user.mention}: command not found.") return + if isinstance(user, discord.User): # For example because a message was sent in a DM. + member = await get_or_fetch_member(channel.guild, user.id) + if member: + user = member ctx = FakeContext(channel, command) if self.name in ("KICK", "WARNING", "WATCH", "NOTE"): await command(ctx, user, reason=reason or None) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index e71bab0d16..98741a96d3 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -11,6 +11,7 @@ from botcore.site_api import ResponseCodeError from botcore.utils import scheduling from botcore.utils.logging import get_logger +from botcore.utils.members import get_or_fetch_member from discord import Embed, Interaction from discord.ext.commands import Context from discord.ui.select import MISSING as SELECT_MISSING, SelectOption @@ -529,7 +530,11 @@ async def user_info(self, interaction: Interaction, button: discord.ui.Button) - await interaction.response.defer() fake_ctx = FakeContext(interaction.channel, command, author=interaction.user) - await command(fake_ctx, self.ctx.author) + # Get the most updated user/member object every time the button is pressed. + author = await get_or_fetch_member(interaction.guild, self.ctx.author.id) + if author is None: + author = await bot.instance.fetch_user(self.ctx.author.id) + await command(fake_ctx, author) @discord.ui.button(emoji="⚠") async def user_infractions(self, interaction: Interaction, button: discord.ui.Button) -> None: From b7297dedbea33a7029d7aee4b69b55fdae801487 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 20 Jan 2023 17:04:18 +0200 Subject: [PATCH 083/132] Show infraction name in passive form in alert --- .../actions/infraction_and_notification.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index d60bcac5ba..215afabdd4 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -18,6 +18,17 @@ log = get_logger(__name__) +passive_form = { + "BAN": "banned", + "KICK": "kicked", + "MUTE": "muted", + "VOICE_MUTE": "voice muted", + "SUPERSTAR": "superstarred", + "WARNING": "warned", + "WATCH": "watch", + "NOTE": "noted", +} + class Infraction(Enum): """An enumeration of infraction types. The lower the value, the higher it is on the hierarchy.""" @@ -146,7 +157,7 @@ async def action(self, ctx: FilterContext) -> None: await self.infraction_type.invoke( ctx.author, channel, alerts_channel, self.infraction_duration, self.infraction_reason ) - ctx.action_descriptions.append(self.infraction_type.name.lower()) + ctx.action_descriptions.append(passive_form[self.infraction_type.name]) def __or__(self, other: ActionEntry): """ From 7015cfc7a5697e454bafdd8d2ed7e77160544b2b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 20 Jan 2023 18:08:24 +0200 Subject: [PATCH 084/132] Render list elements without quotes --- bot/exts/filtering/_ui/ui.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 98741a96d3..3cb73bf8b2 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -118,9 +118,10 @@ def populate_embed_from_dict(embed: Embed, data: dict) -> None: for setting, value in data.items(): if setting.startswith("_"): continue - if type(value) in (set, tuple): - value = list(value) - value = str(value) if value not in ("", None) else "-" + if isinstance(value, (list, set, tuple)): + value = f"[{', '.join(value)}]" + else: + value = str(value) if value not in ("", None) else "-" if len(value) > MAX_FIELD_SIZE: value = value[:MAX_FIELD_SIZE] + " [...]" embed.add_field(name=setting, value=value, inline=len(value) < MAX_INLINE_SIZE) @@ -353,7 +354,9 @@ async def apply_removal(self, interaction: Interaction, select: discord.ui.Selec if _i != len(self.stored_value): self.stored_value.pop(_i) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self.copy()) + await interaction.response.edit_message( + content=f"Current list: [{', '.join(self.stored_value)}]", view=self.copy() + ) self.stop() async def apply_addition(self, interaction: Interaction, item: str) -> None: @@ -363,7 +366,9 @@ async def apply_addition(self, interaction: Interaction, item: str) -> None: return self.stored_value.append(item) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self.copy()) + await interaction.response.edit_message( + content=f"Current list: [{', '.join(self.stored_value)}]", view=self.copy() + ) self.stop() async def apply_edit(self, interaction: Interaction, new_list: str) -> None: @@ -456,7 +461,7 @@ async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.S else: current_list = [] await interaction.response.send_message( - f"Current list: {current_list}", + f"Current list: [{', '.join(current_list)}]", view=SequenceEditView(setting_name, current_list, update_callback), ephemeral=True ) From c3d6a72dcfd436ec8d2133a2acc601339a539633 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 20 Jan 2023 18:56:20 +0200 Subject: [PATCH 085/132] Bring back stats counting for triggered filters --- bot/exts/filtering/filtering.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 8b1a48617d..e0b0e8c463 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -28,7 +28,7 @@ from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filter_lists.filter_list import AtomicList -from bot.exts.filtering._filters.filter import Filter +from bot.exts.filtering._filters.filter import Filter, UniqueFilter from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction from bot.exts.filtering._ui.filter import ( @@ -220,6 +220,7 @@ async def on_message(self, msg: Message) -> None: await self._check_bad_name(nick_ctx) await self._maybe_schedule_msg_delete(ctx, result_actions) + self._increment_stats(triggers) @Cog.listener() async def on_message_edit(self, before: discord.Message, after: discord.Message) -> None: @@ -242,6 +243,7 @@ async def on_message_edit(self, before: discord.Message, after: discord.Message) if ctx.send_alert: await self._send_alert(ctx, list_messages) await self._maybe_schedule_msg_delete(ctx, result_actions) + self._increment_stats(triggers) @Cog.listener() async def on_voice_state_update(self, member: discord.Member, *_) -> None: @@ -264,6 +266,7 @@ async def filter_snekbox_output(self, snekbox_result: str, msg: Message) -> bool await result_actions.action(ctx) if ctx.send_alert: await self._send_alert(ctx, list_messages) + self._increment_stats(triggers) return result_actions is not None @@ -927,6 +930,13 @@ async def _send_alert(self, ctx: FilterContext, triggered_filters: dict[FilterLi username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10], view=AlertView(ctx) ) + def _increment_stats(self, triggered_filters: dict[AtomicList, list[Filter]]) -> None: + """Increment the stats for every filter triggered.""" + for filters in triggered_filters.values(): + for filter_ in filters: + if isinstance(filter_, UniqueFilter): + self.bot.stats.incr(f"filters.{filter_.name}") + async def _recently_alerted_name(self, member: discord.Member) -> bool: """When it hasn't been `HOURS_BETWEEN_NICKNAME_ALERTS` since last alert, return False, otherwise True.""" if last_alert := await self.name_alerts.get(member.id): @@ -952,13 +962,14 @@ async def _check_bad_name(self, ctx: FilterContext) -> None: names_to_check = (name, normalised_name, cleaned_normalised_name) new_ctx = ctx.replace(content=" ".join(names_to_check)) - result_actions, list_messages, _ = await self._resolve_action(new_ctx) + result_actions, list_messages, triggers = await self._resolve_action(new_ctx) if result_actions: await result_actions.action(ctx) if ctx.send_alert: await self._send_alert(ctx, list_messages) # `ctx` has the original content. # Update time when alert sent await self.name_alerts.set(ctx.author.id, arrow.utcnow().timestamp()) + self._increment_stats(triggers) async def _resolve_list_type_and_name( self, ctx: Context, list_type: ListType | None = None, list_name: str | None = None, *, exclude: str = "" From 71f6fe61ab3666ccb68fd1254f337675db0589ef Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 20 Jan 2023 19:00:24 +0200 Subject: [PATCH 086/132] Change CODEOWNERS --- .github/CODEOWNERS | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0bc2bb7933..2b3e422da1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,5 @@ # Extensions **/bot/exts/backend/sync/** @MarkKoz -**/bot/exts/filters/*token_remover.py @MarkKoz **/bot/exts/moderation/*silence.py @MarkKoz bot/exts/info/codeblock/** @MarkKoz bot/exts/utils/extensions.py @MarkKoz @@ -9,14 +8,11 @@ bot/exts/help_channels/** @MarkKoz bot/exts/moderation/** @mbaruh @Den4200 @ks129 @jb3 bot/exts/info/** @Den4200 @jb3 bot/exts/info/information.py @mbaruh @jb3 -bot/exts/filters/** @mbaruh @jb3 +bot/exts/filtering/** @mbaruh bot/exts/fun/** @ks129 bot/exts/utils/** @ks129 @jb3 bot/exts/recruitment/** @wookie184 -# Rules -bot/rules/** @mbaruh - # Utils bot/utils/function.py @MarkKoz bot/utils/lock.py @MarkKoz From 8dd108474f5ff052a42e4b9d90f5a0a395ef3b8d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 21 Jan 2023 21:17:30 +0200 Subject: [PATCH 087/132] Improve logging --- bot/exts/filtering/_ui/ui.py | 2 ++ bot/exts/filtering/filtering.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 3cb73bf8b2..6100882173 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -103,6 +103,8 @@ async def build_mod_alert(ctx: FilterContext, triggered_filters: dict[FilterList actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") mod_alert_message = "\n".join(part for part in (triggered_by, triggered_in, filters, matches, actions) if part) + log.debug(f"{ctx.event.name} Filter:\n{mod_alert_message}") + if ctx.message: mod_alert_message += f"\n**[Original Content]({ctx.message.jump_url})**:\n" else: diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index e0b0e8c463..d91625eda6 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -73,7 +73,7 @@ def __init__(self, bot: Bot): self.filter_lists: dict[str, FilterList] = {} self._subscriptions: defaultdict[Event, list[FilterList]] = defaultdict(list) self.delete_scheduler = scheduling.Scheduler(self.__class__.__name__) - self.webhook: discord.Webhook = None + self.webhook: discord.Webhook | None = None self.loaded_settings = {} self.loaded_filters = {} @@ -89,6 +89,7 @@ async def cog_load(self) -> None: """ await self.bot.wait_until_guild_available() + log.trace("Loading filtering information from the database.") raw_filter_lists = await self.bot.api_client.get("bot/filter/filter_lists") example_list = None for raw_filter_list in raw_filter_lists: @@ -544,6 +545,7 @@ async def f_delete(self, ctx: Context, filter_id: int) -> None: async def delete_list() -> None: """The actual removal routine.""" await bot.instance.api_client.delete(f'bot/filter/filters/{filter_id}') + log.info(f"Successfully deleted filter with ID {filter_id}.") filter_list[list_type].filters.pop(filter_id) await ctx.reply(f"✅ Deleted filter: {filter_}") @@ -827,6 +829,7 @@ async def delete_list() -> None: self.unsubscribe(filter_list) await bot.instance.api_client.delete(f"bot/filter/filter_lists/{list_id}") + log.info(f"Successfully deleted the {filter_list[list_type].label} filterlist.") await message.edit(content=f"✅ The {list_description} list has been deleted.") result = await self._resolve_list_type_and_name(ctx, list_type, list_name) @@ -1163,6 +1166,7 @@ async def _post_new_filter( } response = await bot.instance.api_client.post('bot/filter/filters', json=to_serializable(payload)) new_filter = filter_list.add_filter(list_type, response) + log.info(f"Added new filter: {new_filter}.") if new_filter: await self._maybe_alert_auto_infraction(filter_list, list_type, new_filter) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, new_filter) @@ -1205,6 +1209,7 @@ async def _patch_filter( ) # Return type can be None, but if it's being edited then it's not supposed to be. edited_filter = filter_list.add_filter(list_type, response) + log.info(f"Successfully patched filter {edited_filter}.") await self._maybe_alert_auto_infraction(filter_list, list_type, edited_filter, filter_) extra_msg = Filtering._identical_filters_message(content, filter_list, list_type, edited_filter) await msg.reply(f"✅ Edited filter: {edited_filter}" + extra_msg) @@ -1212,9 +1217,11 @@ async def _patch_filter( async def _post_filter_list(self, msg: Message, list_name: str, list_type: ListType, settings: dict) -> None: """POST the new data of the filter list to the site API.""" payload = {"name": list_name, "list_type": list_type.value, **to_serializable(settings)} + filterlist_name = f"{past_tense(list_type.name.lower())} {list_name}" response = await bot.instance.api_client.post('bot/filter/filter_lists', json=payload) + log.info(f"Successfully posted the new {filterlist_name} filterlist.") self._load_raw_filter_list(response) - await msg.reply(f"✅ Added a new filter list: {past_tense(list_type.name.lower())} {list_name}") + await msg.reply(f"✅ Added a new filter list: {filterlist_name}") @staticmethod async def _patch_filter_list(msg: Message, filter_list: FilterList, list_type: ListType, settings: dict) -> None: @@ -1223,6 +1230,7 @@ async def _patch_filter_list(msg: Message, filter_list: FilterList, list_type: L response = await bot.instance.api_client.patch( f'bot/filter/filter_lists/{list_id}', json=to_serializable(settings) ) + log.info(f"Successfully patched the {filter_list[list_type].label} filterlist, reloading...") filter_list.pop(list_type, None) filter_list.add_list(response) await msg.reply(f"✅ Edited filter list: {filter_list[list_type].label}") @@ -1356,6 +1364,7 @@ async def send_weekly_auto_infraction_report(self, channel: discord.TextChannel If `channel` is not specified, it is sent to #mod-meta. """ + log.trace("Preparing weekly auto-infraction report.") seven_days_ago = arrow.utcnow().shift(days=-7) if not channel: log.info("Auto-infraction report: the channel to report to is missing.") From 44bf2477d675035bb4024d7cd4fa400d7f2b8942 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 22 Jan 2023 21:21:57 +0200 Subject: [PATCH 088/132] Bring back old system tests --- .../filtering/test_discord_token_filter.py | 276 ++++++++++++++++++ .../exts/filtering/test_extension_filter.py | 139 +++++++++ .../exts/filtering/test_settings_entries.py | 66 ++++- .../{test_filters.py => test_token_filter.py} | 14 +- 4 files changed, 484 insertions(+), 11 deletions(-) create mode 100644 tests/bot/exts/filtering/test_discord_token_filter.py create mode 100644 tests/bot/exts/filtering/test_extension_filter.py rename tests/bot/exts/filtering/{test_filters.py => test_token_filter.py} (72%) diff --git a/tests/bot/exts/filtering/test_discord_token_filter.py b/tests/bot/exts/filtering/test_discord_token_filter.py new file mode 100644 index 0000000000..ef124e6ffb --- /dev/null +++ b/tests/bot/exts/filtering/test_discord_token_filter.py @@ -0,0 +1,276 @@ +import unittest +from re import Match +from unittest import mock +from unittest.mock import MagicMock, patch + +import arrow + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.unique import discord_token +from bot.exts.filtering._filters.unique.discord_token import DiscordTokenFilter, Token +from tests.helpers import MockBot, MockMember, MockMessage, MockTextChannel, autospec + + +class DiscordTokenFilterTests(unittest.IsolatedAsyncioTestCase): + """Tests the DiscordTokenFilter class.""" + + def setUp(self): + """Adds the filter, a bot, and a message to the instance for usage in tests.""" + now = arrow.utcnow().timestamp() + self.filter = DiscordTokenFilter({ + "id": 1, + "content": "discord_token", + "description": None, + "settings": {}, + "additional_field": "{}", # noqa: P103 + "created_at": now, + "updated_at": now + }) + + self.msg = MockMessage(id=555, content="hello world") + self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) + + member = MockMember(id=123) + channel = MockTextChannel(id=345) + self.ctx = FilterContext(Event.MESSAGE, member, channel, "", self.msg) + + def test_extract_user_id_valid(self): + """Should consider user IDs valid if they decode into an integer ID.""" + id_pairs = ( + ("NDcyMjY1OTQzMDYyNDEzMzMy", 472265943062413332), + ("NDc1MDczNjI5Mzk5NTQ3OTA0", 475073629399547904), + ("NDY3MjIzMjMwNjUwNzc3NjQx", 467223230650777641), + ) + + for token_id, user_id in id_pairs: + with self.subTest(token_id=token_id): + result = DiscordTokenFilter.extract_user_id(token_id) + self.assertEqual(result, user_id) + + def test_extract_user_id_invalid(self): + """Should consider non-digit and non-ASCII IDs invalid.""" + ids = ( + ("SGVsbG8gd29ybGQ", "non-digit ASCII"), + ("0J_RgNC40LLQtdGCINC80LjRgA", "cyrillic text"), + ("4pO14p6L4p6C4pG34p264pGl8J-EiOKSj-KCieKBsA", "Unicode digits"), + ("4oaA4oaB4oWh4oWi4Lyz4Lyq4Lyr4LG9", "Unicode numerals"), + ("8J2fjvCdn5nwnZ-k8J2fr_Cdn7rgravvvJngr6c", "Unicode decimals"), + ("{hello}[world]&(bye!)", "ASCII invalid Base64"), + ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), + ) + + for user_id, msg in ids: + with self.subTest(msg=msg): + result = DiscordTokenFilter.extract_user_id(user_id) + self.assertIsNone(result) + + def test_is_valid_timestamp_valid(self): + """Should consider timestamps valid if they're greater than the Discord epoch.""" + timestamps = ( + "XsyRkw", + "Xrim9Q", + "XsyR-w", + "XsySD_", + "Dn9r_A", + ) + + for timestamp in timestamps: + with self.subTest(timestamp=timestamp): + result = DiscordTokenFilter.is_valid_timestamp(timestamp) + self.assertTrue(result) + + def test_is_valid_timestamp_invalid(self): + """Should consider timestamps invalid if they're before Discord epoch or can't be parsed.""" + timestamps = ( + ("B4Yffw", "DISCORD_EPOCH - TOKEN_EPOCH - 1"), + ("ew", "123"), + ("AoIKgA", "42076800"), + ("{hello}[world]&(bye!)", "ASCII invalid Base64"), + ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), + ) + + for timestamp, msg in timestamps: + with self.subTest(msg=msg): + result = DiscordTokenFilter.is_valid_timestamp(timestamp) + self.assertFalse(result) + + def test_is_valid_hmac_valid(self): + """Should consider an HMAC valid if it has at least 3 unique characters.""" + valid_hmacs = ( + "VXmErH7j511turNpfURmb0rVNm8", + "Ysnu2wacjaKs7qnoo46S8Dm2us8", + "sJf6omBPORBPju3WJEIAcwW9Zds", + "s45jqDV_Iisn-symw0yDRrk_jf4", + ) + + for hmac in valid_hmacs: + with self.subTest(msg=hmac): + result = DiscordTokenFilter.is_maybe_valid_hmac(hmac) + self.assertTrue(result) + + def test_is_invalid_hmac_invalid(self): + """Should consider an HMAC invalid if has fewer than 3 unique characters.""" + invalid_hmacs = ( + ("xxxxxxxxxxxxxxxxxx", "Single character"), + ("XxXxXxXxXxXxXxXxXx", "Single character alternating case"), + ("ASFasfASFasfASFASsf", "Three characters alternating-case"), + ("asdasdasdasdasdasdasd", "Three characters one case"), + ) + + for hmac, msg in invalid_hmacs: + with self.subTest(msg=msg): + result = DiscordTokenFilter.is_maybe_valid_hmac(hmac) + self.assertFalse(result) + + async def test_no_trigger_when_no_token(self): + """False should be returned if the message doesn't contain a Discord token.""" + return_value = await self.filter.triggered_on(self.ctx) + + self.assertFalse(return_value) + + @autospec(DiscordTokenFilter, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac") + @autospec("bot.exts.filtering._filters.unique.discord_token", "Token") + @autospec("bot.exts.filtering._filters.unique.discord_token", "TOKEN_RE") + def test_find_token_valid_match( + self, + token_re, + token_cls, + extract_user_id, + is_valid_timestamp, + is_maybe_valid_hmac, + ): + """The first match with a valid user ID, timestamp, and HMAC should be returned as a `Token`.""" + matches = [ + mock.create_autospec(Match, spec_set=True, instance=True), + mock.create_autospec(Match, spec_set=True, instance=True), + ] + tokens = [ + mock.create_autospec(Token, spec_set=True, instance=True), + mock.create_autospec(Token, spec_set=True, instance=True), + ] + + token_re.finditer.return_value = matches + token_cls.side_effect = tokens + extract_user_id.side_effect = (None, True) # The 1st match will be invalid, 2nd one valid. + is_valid_timestamp.return_value = True + is_maybe_valid_hmac.return_value = True + + return_value = DiscordTokenFilter.find_token_in_message(self.msg) + + self.assertEqual(tokens[1], return_value) + + @autospec(DiscordTokenFilter, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac") + @autospec("bot.exts.filtering._filters.unique.discord_token", "Token") + @autospec("bot.exts.filtering._filters.unique.discord_token", "TOKEN_RE") + def test_find_token_invalid_matches( + self, + token_re, + token_cls, + extract_user_id, + is_valid_timestamp, + is_maybe_valid_hmac, + ): + """None should be returned if no matches have valid user IDs, HMACs, and timestamps.""" + token_re.finditer.return_value = [mock.create_autospec(Match, spec_set=True, instance=True)] + token_cls.return_value = mock.create_autospec(Token, spec_set=True, instance=True) + extract_user_id.return_value = None + is_valid_timestamp.return_value = False + is_maybe_valid_hmac.return_value = False + + return_value = DiscordTokenFilter.find_token_in_message(self.msg) + + self.assertIsNone(return_value) + + def test_regex_invalid_tokens(self): + """Messages without anything looking like a token are not matched.""" + tokens = ( + "", + "lemon wins", + "..", + "x.y", + "x.y.", + ".y.z", + ".y.", + "..z", + "x..z", + " . . ", + "\n.\n.\n", + "hellö.world.bye", + "base64.nötbåse64.morebase64", + "19jd3J.dfkm3d.€víł§tüff", + ) + + for token in tokens: + with self.subTest(token=token): + results = discord_token.TOKEN_RE.findall(token) + self.assertEqual(len(results), 0) + + def test_regex_valid_tokens(self): + """Messages that look like tokens should be matched.""" + # Don't worry, these tokens have been invalidated. + tokens = ( + "NDcyMjY1OTQzMDYy_DEzMz-y.XsyRkw.VXmErH7j511turNpfURmb0rVNm8", + "NDcyMjY1OTQzMDYyNDEzMzMy.Xrim9Q.Ysnu2wacjaKs7qnoo46S8Dm2us8", + "NDc1MDczNjI5Mzk5NTQ3OTA0.XsyR-w.sJf6omBPORBPju3WJEIAcwW9Zds", + "NDY3MjIzMjMwNjUwNzc3NjQx.XsySD_.s45jqDV_Iisn-symw0yDRrk_jf4", + ) + + for token in tokens: + with self.subTest(token=token): + results = discord_token.TOKEN_RE.fullmatch(token) + self.assertIsNotNone(results, f"{token} was not matched by the regex") + + def test_regex_matches_multiple_valid(self): + """Should support multiple matches in the middle of a string.""" + token_1 = "NDY3MjIzMjMwNjUwNzc3NjQx.XsyWGg.uFNEQPCc4ePwGh7egG8UicQssz8" + token_2 = "NDcyMjY1OTQzMDYyNDEzMzMy.XsyWMw.l8XPnDqb0lp-EiQ2g_0xVFT1pyc" + message = f"garbage {token_1} hello {token_2} world" + + results = discord_token.TOKEN_RE.finditer(message) + results = [match[0] for match in results] + self.assertCountEqual((token_1, token_2), results) + + @autospec("bot.exts.filtering._filters.unique.discord_token", "LOG_MESSAGE") + def test_format_log_message(self, log_message): + """Should correctly format the log message with info from the message and token.""" + token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + log_message.format.return_value = "Howdy" + + return_value = DiscordTokenFilter.format_log_message(self.msg.author, self.msg.channel, token) + + self.assertEqual(return_value, log_message.format.return_value) + + @patch("bot.instance", MockBot()) + @autospec("bot.exts.filtering._filters.unique.discord_token", "UNKNOWN_USER_LOG_MESSAGE") + @autospec("bot.exts.filtering._filters.unique.discord_token", "get_or_fetch_member") + async def test_format_userid_log_message_unknown(self, get_or_fetch_member, unknown_user_log_message): + """Should correctly format the user ID portion when the actual user it belongs to is unknown.""" + token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + unknown_user_log_message.format.return_value = " Partner" + get_or_fetch_member.return_value = None + + return_value = await DiscordTokenFilter.format_userid_log_message(token) + + self.assertEqual(return_value, (unknown_user_log_message.format.return_value, False)) + + @patch("bot.instance", MockBot()) + @autospec("bot.exts.filtering._filters.unique.discord_token", "KNOWN_USER_LOG_MESSAGE") + async def test_format_userid_log_message_bot(self, known_user_log_message): + """Should correctly format the user ID portion when the ID belongs to a known bot.""" + token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + known_user_log_message.format.return_value = " Partner" + + return_value = await DiscordTokenFilter.format_userid_log_message(token) + + self.assertEqual(return_value, (known_user_log_message.format.return_value, True)) + + @patch("bot.instance", MockBot()) + @autospec("bot.exts.filtering._filters.unique.discord_token", "KNOWN_USER_LOG_MESSAGE") + async def test_format_log_message_user_token_user(self, user_token_message): + """Should correctly format the user ID portion when the ID belongs to a known user.""" + token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + user_token_message.format.return_value = "Partner" + + return_value = await DiscordTokenFilter.format_userid_log_message(token) + + self.assertEqual(return_value, (user_token_message.format.return_value, True)) diff --git a/tests/bot/exts/filtering/test_extension_filter.py b/tests/bot/exts/filtering/test_extension_filter.py new file mode 100644 index 0000000000..0ad41116d4 --- /dev/null +++ b/tests/bot/exts/filtering/test_extension_filter.py @@ -0,0 +1,139 @@ +import unittest +from unittest.mock import MagicMock, patch + +import arrow + +from bot.constants import Channels +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filter_lists import extension +from bot.exts.filtering._filter_lists.extension import ExtensionsList +from bot.exts.filtering._filter_lists.filter_list import ListType +from tests.helpers import MockAttachment, MockBot, MockMember, MockMessage, MockTextChannel + +BOT = MockBot() + + +class ExtensionsListTests(unittest.IsolatedAsyncioTestCase): + """Test the ExtensionsList class.""" + + def setUp(self): + """Sets up fresh objects for each test.""" + self.filter_list = ExtensionsList(MagicMock()) + now = arrow.utcnow().timestamp() + filters = [] + self.whitelist = [".first", ".second", ".third"] + for i, filter_content in enumerate(self.whitelist, start=1): + filters.append({ + "id": i, "content": filter_content, "description": None, "settings": {}, + "additional_field": "{}", "created_at": now, "updated_at": now # noqa: P103 + }) + self.filter_list.add_list({ + "id": 1, + "list_type": 1, + "created_at": now, + "updated_at": now, + "settings": {}, + "filters": filters + }) + + self.message = MockMessage() + member = MockMember(id=123) + channel = MockTextChannel(id=345) + self.ctx = FilterContext(Event.MESSAGE, member, channel, "", self.message) + + @patch("bot.instance", BOT) + async def test_message_with_allowed_attachment(self): + """Messages with allowed extensions should trigger the whitelist and result in no actions or messages.""" + attachment = MockAttachment(filename="python.first") + self.message.attachments = [attachment] + + result = await self.filter_list.actions_for(self.ctx) + + self.assertEqual(result, (None, [], {ListType.ALLOW: [self.filter_list[ListType.ALLOW].filters[1]]})) + + @patch("bot.instance", BOT) + async def test_message_without_attachment(self): + """Messages without attachments should return no triggers, messages, or actions.""" + result = await self.filter_list.actions_for(self.ctx) + + self.assertEqual(result, (None, [], {})) + + @patch("bot.instance", BOT) + async def test_message_with_illegal_extension(self): + """A message with an illegal extension shouldn't trigger the whitelist, and return some action and message.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + + result = await self.filter_list.actions_for(self.ctx) + + self.assertEqual(result, ({}, ["`.disallowed`"], {ListType.ALLOW: []})) + + @patch("bot.instance", BOT) + async def test_python_file_redirect_embed_description(self): + """A message containing a .py file should result in an embed redirecting the user to our paste site.""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + + await self.filter_list.actions_for(self.ctx) + + self.assertEqual(self.ctx.dm_embed, extension.PY_EMBED_DESCRIPTION) + + @patch("bot.instance", BOT) + async def test_txt_file_redirect_embed_description(self): + """A message containing a .txt/.json/.csv file should result in the correct embed.""" + test_values = ( + ("text", ".txt"), + ("json", ".json"), + ("csv", ".csv"), + ) + + for file_name, disallowed_extension in test_values: + with self.subTest(file_name=file_name, disallowed_extension=disallowed_extension): + + attachment = MockAttachment(filename=f"{file_name}{disallowed_extension}") + self.message.attachments = [attachment] + + await self.filter_list.actions_for(self.ctx) + + self.assertEqual( + self.ctx.dm_embed, + extension.TXT_EMBED_DESCRIPTION.format( + blocked_extension=disallowed_extension, + ) + ) + + @patch("bot.instance", BOT) + async def test_other_disallowed_extension_embed_description(self): + """Test the description for a non .py/.txt/.json/.csv disallowed extension.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + + await self.filter_list.actions_for(self.ctx) + meta_channel = BOT.get_channel(Channels.meta) + + self.assertEqual( + self.ctx.dm_embed, + extension.DISALLOWED_EMBED_DESCRIPTION.format( + joined_whitelist=", ".join(self.whitelist), + blocked_extensions_str=".disallowed", + meta_channel_mention=meta_channel.mention + ) + ) + + @patch("bot.instance", BOT) + async def test_get_disallowed_extensions(self): + """The return value should include all non-whitelisted extensions.""" + test_values = ( + ([], []), + (self.whitelist, []), + ([".first"], []), + ([".first", ".disallowed"], ["`.disallowed`"]), + ([".disallowed"], ["`.disallowed`"]), + ([".disallowed", ".illegal"], ["`.disallowed`", "`.illegal`"]), + ) + + for extensions, expected_disallowed_extensions in test_values: + with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): + self.message.attachments = [MockAttachment(filename=f"filename{ext}") for ext in extensions] + result = await self.filter_list.actions_for(self.ctx) + self.assertCountEqual(result[1], expected_disallowed_extensions) diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py index 34b155d6b6..5a1eb6fe63 100644 --- a/tests/bot/exts/filtering/test_settings_entries.py +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -50,7 +50,9 @@ def test_role_bypass_is_on_for_a_user_with_the_right_role(self): def test_context_doesnt_trigger_for_empty_channel_scope(self): """A filter is enabled for all channels by default.""" channel = MockTextChannel() - scope = ChannelScope(disabled_channels=None, disabled_categories=None, enabled_channels=None) + scope = ChannelScope( + disabled_channels=None, disabled_categories=None, enabled_channels=None, enabled_categories=None + ) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -60,7 +62,9 @@ def test_context_doesnt_trigger_for_empty_channel_scope(self): def test_context_doesnt_trigger_for_disabled_channel(self): """A filter shouldn't trigger if it's been disabled in the channel.""" channel = MockTextChannel(id=123) - scope = ChannelScope(disabled_channels=["123"], disabled_categories=None, enabled_channels=None) + scope = ChannelScope( + disabled_channels=["123"], disabled_categories=None, enabled_channels=None, enabled_categories=None + ) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -70,7 +74,9 @@ def test_context_doesnt_trigger_for_disabled_channel(self): def test_context_doesnt_trigger_in_disabled_category(self): """A filter shouldn't trigger if it's been disabled in the category.""" channel = MockTextChannel(category=MockCategoryChannel(id=456)) - scope = ChannelScope(disabled_channels=None, disabled_categories=["456"], enabled_channels=None) + scope = ChannelScope( + disabled_channels=None, disabled_categories=["456"], enabled_channels=None, enabled_categories=None + ) self.ctx.channel = channel result = scope.triggers_on(self.ctx) @@ -80,13 +86,51 @@ def test_context_doesnt_trigger_in_disabled_category(self): def test_context_triggers_in_enabled_channel_in_disabled_category(self): """A filter should trigger in an enabled channel even if it's been disabled in the category.""" channel = MockTextChannel(id=123, category=MockCategoryChannel(id=234)) - scope = ChannelScope(disabled_channels=None, disabled_categories=["234"], enabled_channels=["123"]) + scope = ChannelScope( + disabled_channels=None, disabled_categories=["234"], enabled_channels=["123"], enabled_categories=None + ) + self.ctx.channel = channel + + result = scope.triggers_on(self.ctx) + + self.assertTrue(result) + + def test_context_triggers_inside_enabled_category(self): + """A filter shouldn't trigger outside enabled categories, if there are any.""" + channel = MockTextChannel(id=123, category=MockCategoryChannel(id=234)) + scope = ChannelScope( + disabled_channels=None, disabled_categories=None, enabled_channels=None, enabled_categories=["234"] + ) self.ctx.channel = channel result = scope.triggers_on(self.ctx) self.assertTrue(result) + def test_context_doesnt_trigger_outside_enabled_category(self): + """A filter shouldn't trigger outside enabled categories, if there are any.""" + channel = MockTextChannel(id=123, category=MockCategoryChannel(id=234)) + scope = ChannelScope( + disabled_channels=None, disabled_categories=None, enabled_channels=None, enabled_categories=["789"] + ) + self.ctx.channel = channel + + result = scope.triggers_on(self.ctx) + + self.assertFalse(result) + + def test_context_doesnt_trigger_inside_disabled_channel_in_enabled_category(self): + """A filter shouldn't trigger outside enabled categories, if there are any.""" + channel = MockTextChannel(id=123, category=MockCategoryChannel(id=234)) + scope = ChannelScope( + disabled_channels=["123"], disabled_categories=None, enabled_channels=None, enabled_categories=["234"] + ) + self.ctx.channel = channel + + result = scope.triggers_on(self.ctx) + + self.assertFalse(result) + def test_filtering_dms_when_necessary(self): """A filter correctly ignores or triggers in a channel depending on the value of FilterDM.""" cases = ( @@ -112,14 +156,16 @@ def test_infraction_merge_of_same_infraction_type(self): infraction_reason="hi", infraction_duration=10, dm_content="how", - dm_embed="what is" + dm_embed="what is", + infraction_channel=0 ) infraction2 = InfractionAndNotification( infraction_type="MUTE", infraction_reason="there", infraction_duration=20, dm_content="are you", - dm_embed="your name" + dm_embed="your name", + infraction_channel=0 ) result = infraction1 | infraction2 @@ -132,6 +178,7 @@ def test_infraction_merge_of_same_infraction_type(self): "infraction_duration": 20.0, "dm_content": "are you", "dm_embed": "your name", + "infraction_channel": 0 } ) @@ -142,14 +189,16 @@ def test_infraction_merge_of_different_infraction_types(self): infraction_reason="hi", infraction_duration=20, dm_content="", - dm_embed="" + dm_embed="", + infraction_channel=0 ) infraction2 = InfractionAndNotification( infraction_type="BAN", infraction_reason="", infraction_duration=10, dm_content="there", - dm_embed="" + dm_embed="", + infraction_channel=0 ) result = infraction1 | infraction2 @@ -162,5 +211,6 @@ def test_infraction_merge_of_different_infraction_types(self): "infraction_duration": 10.0, "dm_content": "there", "dm_embed": "", + "infraction_channel": 0 } ) diff --git a/tests/bot/exts/filtering/test_filters.py b/tests/bot/exts/filtering/test_token_filter.py similarity index 72% rename from tests/bot/exts/filtering/test_filters.py rename to tests/bot/exts/filtering/test_token_filter.py index 29b50188a2..0dfc8ae9fe 100644 --- a/tests/bot/exts/filtering/test_filters.py +++ b/tests/bot/exts/filtering/test_token_filter.py @@ -1,11 +1,13 @@ import unittest +import arrow + from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filters.token import TokenFilter from tests.helpers import MockMember, MockMessage, MockTextChannel -class FilterTests(unittest.IsolatedAsyncioTestCase): +class TokenFilterTests(unittest.IsolatedAsyncioTestCase): """Test functionality of the token filter.""" def setUp(self) -> None: @@ -20,8 +22,12 @@ async def test_token_filter_triggers(self): (r"hi", "oh hi there", True), (r"hi", "goodbye", False), (r"bla\d{2,4}", "bla18", True), - (r"bla\d{2,4}", "bla1", False) + (r"bla\d{2,4}", "bla1", False), + # See advisory https://github.com/python-discord/bot/security/advisories/GHSA-j8c3-8x46-8pp6 + (r"TOKEN", "https://google.com TOKEN", True), + (r"TOKEN", "https://google.com something else", False) ) + now = arrow.utcnow().timestamp() for pattern, content, expected in test_cases: with self.subTest( @@ -34,7 +40,9 @@ async def test_token_filter_triggers(self): "content": pattern, "description": None, "settings": {}, - "additional_field": "{}" # noqa: P103 + "additional_field": "{}", # noqa: P103 + "created_at": now, + "updated_at": now }) self.ctx.content = content result = await filter_.triggered_on(self.ctx) From 858205c712b3a9c677b5162f36845e3c7473075e Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Jan 2023 16:30:53 +0200 Subject: [PATCH 089/132] Post descriptive embed on unexpected error --- bot/exts/filtering/_ui/ui.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 6100882173..f0055d72df 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -156,6 +156,8 @@ def format_response_error(e: ResponseCodeError) -> Embed: description = description.strip() if len(description) > MAX_EMBED_DESCRIPTION: description = description[:MAX_EMBED_DESCRIPTION] + "[...]" + if not description: + description = "Something unexpected happened, check the logs." embed = Embed(colour=discord.Colour.red(), title="Oops...", description=description) return embed From 4626b07e2750cc6dbf1e02b33dfc0015d6a8fdf9 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Jan 2023 22:59:41 +0200 Subject: [PATCH 090/132] Fix display of sequence setting with non-string elements --- bot/exts/filtering/_ui/ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index f0055d72df..bc9fc09f03 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -121,7 +121,7 @@ def populate_embed_from_dict(embed: Embed, data: dict) -> None: if setting.startswith("_"): continue if isinstance(value, (list, set, tuple)): - value = f"[{', '.join(value)}]" + value = f"[{', '.join(map(str, value))}]" else: value = str(value) if value not in ("", None) else "-" if len(value) > MAX_FIELD_SIZE: From 8c13be031ab52b9e0392a0766f57b2b0aeb18726 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Jan 2023 23:00:37 +0200 Subject: [PATCH 091/132] Include filterlist information when patching filters The information is necessary for the serializer validaiton. --- bot/exts/filtering/filtering.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index d91625eda6..a66adfc731 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1200,9 +1200,12 @@ async def _patch_filter( for setting_entry in current_settings.values(): settings.update({setting: None for setting in setting_entry.dict() if setting not in settings}) + # Even though the list ID remains unchanged, it still needs to be provided for correct serializer validation. + list_id = filter_list[list_type].id description = description or None payload = { - "content": content, "description": description, "additional_field": json.dumps(filter_settings), **settings + "filter_list": list_id, "content": content, "description": description, + "additional_field": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.patch( f'bot/filter/filters/{filter_.id}', json=to_serializable(payload) From 34fd3e75ab3cd7951944d9d49ca66fa3b15e1b02 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 28 Jan 2023 03:13:45 +0200 Subject: [PATCH 092/132] Handle response message being a list --- bot/exts/filtering/_ui/ui.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index bc9fc09f03..dc996faf72 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -148,11 +148,15 @@ def parse_value(value: str, type_: type[T]) -> T: def format_response_error(e: ResponseCodeError) -> Embed: """Format the response error into an embed.""" description = "" - if "non_field_errors" in e.response_json: - non_field_errors = e.response_json.pop("non_field_errors") - description += "\n".join(f"• {error}" for error in non_field_errors) + "\n" - for field, errors in e.response_json.items(): - description += "\n".join(f"• {field} - {error}" for error in errors) + "\n" + if isinstance(e.response_json, list): + description = "\n".join(f"• {error}" for error in e.response_json) + elif isinstance(e.response_json, dict): + if "non_field_errors" in e.response_json: + non_field_errors = e.response_json.pop("non_field_errors") + description += "\n".join(f"• {error}" for error in non_field_errors) + "\n" + for field, errors in e.response_json.items(): + description += "\n".join(f"• {field} - {error}" for error in errors) + "\n" + description = description.strip() if len(description) > MAX_EMBED_DESCRIPTION: description = description[:MAX_EMBED_DESCRIPTION] + "[...]" From dc0ce05ff1406954f5006366b2899c4bde1967ce Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 28 Jan 2023 18:11:36 +0200 Subject: [PATCH 093/132] Create a guide for developing the filtering ext --- bot/exts/filtering/FILTERS-DEVELOPMENT.md | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 bot/exts/filtering/FILTERS-DEVELOPMENT.md diff --git a/bot/exts/filtering/FILTERS-DEVELOPMENT.md b/bot/exts/filtering/FILTERS-DEVELOPMENT.md new file mode 100644 index 0000000000..d5896d5565 --- /dev/null +++ b/bot/exts/filtering/FILTERS-DEVELOPMENT.md @@ -0,0 +1,63 @@ +# Filters Development +This file gives a short overview of the extension, and shows how to perform some basic changes/additions to it. + +## Overview +The main idea is that there is a list of filters each deciding whether they apply to the given content. +For example, there can be a filter that decides it will trigger when the content contains the string "lemon". + +There are several types of filters, and two filters of the same type differ by their content. +For example, filters of type "token" search for a specific token inside the provided string. +One token filter might look for the string "lemon", while another will look for the string "joe". + +Each filter has a set of settings that decide when it triggers (e.g. in which channels), and what happens if it does (e.g. delete the message). +Filters of a specific type can have additional settings that are special to them. + +A list of filters is contained within a filter list. +The filter list gets content to filter, and dispatches it to each of its filters. +It takes the answers from its filters and returns a unified response (e.g. if at least one of the filters says it should be deleted, then the filter list response will include it). + +A filter list has the same set of possible settings, which act as defaults. +If a filter in the list doesn't define a value for a setting (meaning it has a value of None), it will use the value of the containing filter list. + +The cog receives "filtering events". For example, a new message is sent. +It creates a "filtering context" with everything a filtering list needs to know to provide an answer for what should be done. +For example, if the event is a new message, then the content to filter is the content of the message, embeds if any exist, etc. + +The cog dispatches the event to each filter list, gets the result from each, compiles them, and takes any action dictated by them. +For example, if any of the filter lists want the message to be deleted, then the cog will delete it. + +## Example Changes +### Creating a new type of filter list +1. Head over to `bot.exts.filtering._filter_lists` and create a new Python file. +2. Subclass the FilterList class in `bot.exts.filtering._filter_lists.filter_list` and implement its abstract methods. Make sure to set the `name` class attribute. + +You can now add filter lists to the database with the same name defined in the new FilterList subclass. + +### Creating a new type of filter +1. Head over to `bot.exts.filtering._filters` and create a new Python file. +2. Subclass the Filter class in `bot.exts.filtering._filters.filter` and implement its abstract methods. +3. Make sure to set the `name` class attribute, and have one of the FilterList subclasses return this new Filter subclass in `get_filter_type`. + +### Creating a new type of setting +1. Head over to `bot.exts.filtering._settings_types`, and open a new Python file in either `actions` or `validations`, depending on whether you want to subclass `ActionEntry` or `ValidationEntry`. +2. Subclass one of the aforementioned classes, and implement its abstract methods. Make sure to set the `name` and `description` class attributes. + +You can now make the appropriate changes to the site repo: +1. Add a new field in the `Filter` and `FilterList` models. Make sure that on `Filter` it's nullable, and on `FilterList` it isn't. +2. In `serializers.py`, add the new field to `SETTINGS_FIELDS`, and to `ALLOW_BLANK_SETTINGS` or `ALLOW_EMPTY_SETTINGS` if appropriate. If it's not a part of any group of settings, add it `BASE_SETTINGS_FIELDS`, otherwise add it to the appropriate group or create a new one. +3. If you created a new group, make sure it's used in `to_representation`. +4. Update the docs in the filter viewsets. + +You can merge the changes to the bot first - if no such field is loaded from the database it'll just be ignored. + +You can define entries that are a group of fields in the database. +In that case the created subclass should have fields whose names are the names of the fields in the database. +Then, the description will be a dictionary, whose keys are the names of the fields, and values are the descriptions for each field. + +### Creating a new type of filtering event +1. Head over to `bot.exts.filtering._filter_context` and add a new value to the `Event` enum. +2. Implement the dispatching and actioning of the new event in the cog, by either adding it to an existing even listener, or creating a new one. +3. Have the appropriate filter lists subscribe to the event, so they receive it. +4. Have the appropriate unique filters (currently under `unique` and `antispam` in `bot.exts.filtering._filters`) subscribe to the event, so they receive it. + +It should be noted that the filtering events don't need to correspond to Discord events. For example, `nickname` isn't a Discord event and is dispatched when a message is sent. From 12e1e3771309bf7a39131687984d54859c691c9d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 27 Feb 2023 21:53:41 +0200 Subject: [PATCH 094/132] Simplify `Settings.copy` Co-authored-by: Ionite --- bot/exts/filtering/_settings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index a752918a62..fff4239ec9 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -3,6 +3,7 @@ import operator import traceback from abc import abstractmethod +from copy import copy from functools import reduce from typing import Any, NamedTuple, Optional, TypeVar @@ -105,9 +106,7 @@ def overrides(self) -> dict[str, Any]: def copy(self: TSettings) -> TSettings: """Create a shallow copy of the object.""" - copy = self.__class__({}) - copy.update(super().copy()) # Copy the internal dict. - return copy + return copy(self) def get_setting(self, key: str, default: Optional[Any] = None) -> Any: """Get the setting matching the key, or fall back to the default value if the key is missing.""" From 333bbf630164a782df9f375f6d343d80704f1711 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 27 Feb 2023 22:09:46 +0200 Subject: [PATCH 095/132] Use public typing API for `starting_value` Co-authored-by: Ionite --- bot/exts/filtering/_utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index 5abb2240b8..b96767fbf1 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -2,11 +2,12 @@ import importlib.util import inspect import pkgutil +import types from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass from functools import cache -from typing import Any, Iterable, TypeVar, Union +from typing import Any, Iterable, TypeVar, Union, get_args, get_origin import discord import regex @@ -121,13 +122,13 @@ def repr_equals(override: Any, default: Any) -> bool: def starting_value(type_: type[T]) -> T: """Return a value of the given type.""" - if hasattr(type_, "__origin__"): - if type_.__origin__ is not Union: # In case this is a types.GenericAlias or a typing._GenericAlias - type_ = type_.__origin__ - if hasattr(type_, "__args__"): # In case of a Union - if type(None) in type_.__args__: + if get_origin(type_) in (Union, types.UnionType): # In case of a Union + args = get_args(type_) + if type(None) in args: return None - type_ = type_.__args__[0] # Pick one, doesn't matter + type_ = args[0] # Pick one, doesn't matter + if origin := get_origin(type_): # In case of a parameterized List, Set, Dict etc. + type_ = origin try: return type_() From b67e849eff799e9d93b0c0e1909568db5a50770b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 27 Feb 2023 22:50:08 +0200 Subject: [PATCH 096/132] Explicitly handle embed being too long Co-authored-by: Ionite --- bot/exts/filtering/_ui/filter.py | 8 +++++--- bot/exts/filtering/_ui/filter_list.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index e41c663c58..1534aa122d 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -12,8 +12,8 @@ from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._ui.ui import ( - COMPONENT_TIMEOUT, CustomCallbackSelect, EditBaseView, MISSING, SETTINGS_DELIMITER, SINGLE_SETTING_PATTERN, - format_response_error, parse_value, populate_embed_from_dict + COMPONENT_TIMEOUT, CustomCallbackSelect, EditBaseView, MAX_EMBED_DESCRIPTION, MISSING, SETTINGS_DELIMITER, + SINGLE_SETTING_PATTERN, format_response_error, parse_value, populate_embed_from_dict ) from bot.exts.filtering._utils import repr_equals, to_serializable from bot.log import get_logger @@ -278,6 +278,8 @@ async def update_embed( self.embed.description = f"`{content}`" if content else "*No content*" if description and description is not self._REMOVE: self.embed.description += f" - {description}" + if len(self.embed.description) > MAX_EMBED_DESCRIPTION: + self.embed.description = self.embed.description[:MAX_EMBED_DESCRIPTION - 5] + "[...]" if setting_name: # Find the right dictionary to update. @@ -308,7 +310,7 @@ async def update_embed( await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) else: await interaction_or_msg.edit(embed=self.embed, view=new_view) - except discord.errors.HTTPException: # Various errors such as embed description being too long. + except discord.errors.HTTPException: # Various unexpected errors. pass else: self.stop() diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index e77e29ec91..5a0b1c44ab 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -145,7 +145,7 @@ async def update_embed( await interaction_or_msg.response.edit_message(embed=self.embed, view=new_view) else: await interaction_or_msg.edit(embed=self.embed, view=new_view) - except discord.errors.HTTPException: # Various errors such as embed description being too long. + except discord.errors.HTTPException: # Various unexpected errors. pass else: self.stop() From 8d108bde815f43f24c4a16b231b12505a0b1e479 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 13:07:17 +0200 Subject: [PATCH 097/132] `__or__` -> `union` + `Self` type-hint in action settings Keeps the type hint accurate in the subclasses, and avoids having type guards for each implementation. Co-authored-by: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Co-authored-by: Ionite --- bot/exts/filtering/_filter_lists/filter_list.py | 3 +-- bot/exts/filtering/_filter_lists/invite.py | 2 +- bot/exts/filtering/_settings.py | 6 ++++-- .../_settings_types/actions/infraction_and_notification.py | 6 ++---- bot/exts/filtering/_settings_types/actions/ping.py | 6 ++---- .../filtering/_settings_types/actions/remove_context.py | 6 ++---- bot/exts/filtering/_settings_types/actions/send_alert.py | 7 +++---- bot/exts/filtering/_settings_types/settings_entry.py | 3 ++- bot/exts/filtering/filtering.py | 3 +-- 9 files changed, 18 insertions(+), 24 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index a32b0741e2..4913315ebb 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -6,7 +6,6 @@ from dataclasses import dataclass from enum import Enum from functools import reduce -from operator import or_ from typing import Any import arrow @@ -133,7 +132,7 @@ def merge_actions(self, filters: list[Filter]) -> ActionSettings | None: return None try: return reduce( - or_, (filter_.actions or self.defaults.actions for filter_ in filters) + ActionSettings.union, (filter_.actions or self.defaults.actions for filter_ in filters) ).fallback_to(self.defaults.actions) except TypeError: # The sequence fed to reduce is empty, meaning none of the filters have actions, diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index dd14d22223..c2a75e8310 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -120,7 +120,7 @@ async def actions_for( # Blocked invites come second so that their actions have preference. if triggered: if actions: - actions |= self[ListType.DENY].merge_actions(triggered) + actions = actions.union(self[ListType.DENY].merge_actions(triggered)) else: actions = self[ListType.DENY].merge_actions(triggered) all_triggers[ListType.DENY] = triggered diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index fff4239ec9..75e810df59 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -7,6 +7,8 @@ from functools import reduce from typing import Any, NamedTuple, Optional, TypeVar +from typing_extensions import Self + from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types import settings_types from bot.exts.filtering._settings_types.settings_entry import ActionEntry, SettingsEntry, ValidationEntry @@ -175,13 +177,13 @@ class ActionSettings(Settings[ActionEntry]): def __init__(self, settings_data: dict, *, defaults: Settings | None = None, keep_empty: bool = False): super().__init__(settings_data, defaults=defaults, keep_empty=keep_empty) - def __or__(self, other: ActionSettings) -> ActionSettings: + def union(self, other: Self) -> Self: """Combine the entries of two collections of settings into a new ActionsSettings.""" actions = {} # A settings object doesn't necessarily have all types of entries (e.g in the case of filter overrides). for entry in self: if entry in other: - actions[entry] = self[entry] | other[entry] + actions[entry] = self[entry].union(other[entry]) else: actions[entry] = self[entry] for entry in other: diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 215afabdd4..a3b59d05d2 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -9,6 +9,7 @@ from discord import Colour, Embed, Member, User from discord.errors import Forbidden from pydantic import validator +from typing_extensions import Self import bot as bot_module from bot.constants import Channels @@ -159,7 +160,7 @@ async def action(self, ctx: FilterContext) -> None: ) ctx.action_descriptions.append(passive_form[self.infraction_type.name]) - def __or__(self, other: ActionEntry): + def union(self, other: Self) -> Self: """ Combines two actions of the same type. Each type of action is executed once per filter. @@ -169,9 +170,6 @@ def __or__(self, other: ActionEntry): To avoid bombarding the user with several notifications, the message with the more significant infraction is used. """ - if not isinstance(other, InfractionAndNotification): - return NotImplemented - # Lower number -> higher in the hierarchy if self.infraction_type is None: return other.copy() diff --git a/bot/exts/filtering/_settings_types/actions/ping.py b/bot/exts/filtering/_settings_types/actions/ping.py index b3725917ca..ee40c54fe0 100644 --- a/bot/exts/filtering/_settings_types/actions/ping.py +++ b/bot/exts/filtering/_settings_types/actions/ping.py @@ -1,6 +1,7 @@ from typing import ClassVar from pydantic import validator +from typing_extensions import Self from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry @@ -39,9 +40,6 @@ async def action(self, ctx: FilterContext) -> None: new_content = " ".join([resolve_mention(mention) for mention in mentions]) ctx.alert_content = f"{new_content} {ctx.alert_content}" - def __or__(self, other: ActionEntry): + def union(self, other: Self) -> Self: """Combines two actions of the same type. Each type of action is executed once per filter.""" - if not isinstance(other, Ping): - return NotImplemented - return Ping(guild_pings=self.guild_pings | other.guild_pings, dm_pings=self.dm_pings | other.dm_pings) diff --git a/bot/exts/filtering/_settings_types/actions/remove_context.py b/bot/exts/filtering/_settings_types/actions/remove_context.py index 4107400282..e030f06d2b 100644 --- a/bot/exts/filtering/_settings_types/actions/remove_context.py +++ b/bot/exts/filtering/_settings_types/actions/remove_context.py @@ -5,6 +5,7 @@ from botcore.utils.logging import get_logger from discord import Message from discord.errors import HTTPException +from typing_extensions import Self import bot from bot.constants import Channels @@ -105,9 +106,6 @@ async def _handle_nickname(ctx: FilterContext) -> None: await command(FakeContext(alerts_channel, command), ctx.author, None, reason=SUPERSTAR_REASON) ctx.action_descriptions.append("superstar") - def __or__(self, other: ActionEntry): + def union(self, other: Self) -> Self: """Combines two actions of the same type. Each type of action is executed once per filter.""" - if not isinstance(other, RemoveContext): - return NotImplemented - return RemoveContext(remove_context=self.remove_context or other.remove_context) diff --git a/bot/exts/filtering/_settings_types/actions/send_alert.py b/bot/exts/filtering/_settings_types/actions/send_alert.py index 04e4007643..f554cdd4d0 100644 --- a/bot/exts/filtering/_settings_types/actions/send_alert.py +++ b/bot/exts/filtering/_settings_types/actions/send_alert.py @@ -1,5 +1,7 @@ from typing import ClassVar +from typing_extensions import Self + from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry @@ -16,9 +18,6 @@ async def action(self, ctx: FilterContext) -> None: """Add the stored pings to the alert message content.""" ctx.send_alert = self.send_alert - def __or__(self, other: ActionEntry): + def union(self, other: Self) -> Self: """Combines two actions of the same type. Each type of action is executed once per filter.""" - if not isinstance(other, SendAlert): - return NotImplemented - return SendAlert(send_alert=self.send_alert or other.send_alert) diff --git a/bot/exts/filtering/_settings_types/settings_entry.py b/bot/exts/filtering/_settings_types/settings_entry.py index 31e11108de..e41ef5c7ae 100644 --- a/bot/exts/filtering/_settings_types/settings_entry.py +++ b/bot/exts/filtering/_settings_types/settings_entry.py @@ -4,6 +4,7 @@ from typing import Any, ClassVar, Union from pydantic import BaseModel, PrivateAttr +from typing_extensions import Self from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._utils import FieldRequiring @@ -80,7 +81,7 @@ async def action(self, ctx: FilterContext) -> None: ... @abstractmethod - def __or__(self, other: ActionEntry): + def union(self, other: Self) -> Self: """ Combine two actions of the same type. Each type of action is executed once per filter. diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index a66adfc731..66f4715f31 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1,6 +1,5 @@ import datetime import json -import operator import re import unicodedata from collections import defaultdict @@ -917,7 +916,7 @@ async def _resolve_action( result_actions = None if actions: - result_actions = reduce(operator.or_, (action for action in actions)) + result_actions = reduce(ActionSettings.union, actions) return result_actions, messages, triggers From 2bd9d23046969c53cc800f2af970fdad834be828 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 13:17:46 +0200 Subject: [PATCH 098/132] Properly parameterize type hints in filter context Co-authored-by: decorator-factory <42166884+decorator-factory@users.noreply.github.com> --- bot/exts/filtering/_filter_context.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 61f8c9fbcb..c5ebd39978 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field, replace from enum import Enum, auto -from discord import DMChannel, Member, Message, TextChannel, Thread, User +from discord import DMChannel, Embed, Member, Message, TextChannel, Thread, User from bot.utils.message_cache import MessageCache @@ -31,7 +31,7 @@ class FilterContext: channel: TextChannel | Thread | DMChannel | None # The channel involved content: str | Iterable # What actually needs filtering message: Message | None # The message involved - embeds: list = field(default_factory=list) # Any embeds involved + embeds: list[Embed] = field(default_factory=list) # Any embeds involved before_message: Message | None = None message_cache: MessageCache | None = None # Output context @@ -39,7 +39,7 @@ class FilterContext: dm_embed: str = field(default_factory=str) # The embed description to DM the invoker send_alert: bool = field(default=False) # Whether to send an alert for the moderators alert_content: str = field(default_factory=str) # The content of the alert - alert_embeds: list = field(default_factory=list) # Any embeds to add to the alert + alert_embeds: list[Embed] = field(default_factory=list) # Any embeds to add to the alert action_descriptions: list = field(default_factory=list) # What actions were taken matches: list[str] = field(default_factory=list) # What exactly was found notification_domain: str = field(default_factory=str) # A domain to send the user for context From ddcf2c6e122c5f58c00bca7bc7dda98316c4d3ba Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 13:23:23 +0200 Subject: [PATCH 099/132] Use literals as defaults where possible Co-authored-by: wookie184 --- bot/exts/filtering/_filter_context.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index c5ebd39978..9faa16b533 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -35,14 +35,14 @@ class FilterContext: before_message: Message | None = None message_cache: MessageCache | None = None # Output context - dm_content: str = field(default_factory=str) # The content to DM the invoker - dm_embed: str = field(default_factory=str) # The embed description to DM the invoker - send_alert: bool = field(default=False) # Whether to send an alert for the moderators - alert_content: str = field(default_factory=str) # The content of the alert + dm_content: str = "" # The content to DM the invoker + dm_embed: str = "" # The embed description to DM the invoker + send_alert: bool = False # Whether to send an alert for the moderators + alert_content: str = "" # The content of the alert alert_embeds: list[Embed] = field(default_factory=list) # Any embeds to add to the alert action_descriptions: list = field(default_factory=list) # What actions were taken matches: list[str] = field(default_factory=list) # What exactly was found - notification_domain: str = field(default_factory=str) # A domain to send the user for context + notification_domain: str = "" # A domain to send the user for context filter_info: dict['Filter', str] = field(default_factory=dict) # Additional info from a filter. # Additional actions to perform additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list) From 7184925179775a433a5d7982ed7904aabc4a3a56 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 13:45:47 +0200 Subject: [PATCH 100/132] Fix Infraction check Co-authored-by: decorator-factory <42166884+decorator-factory@users.noreply.github.com> --- .../_settings_types/actions/infraction_and_notification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index a3b59d05d2..389d2b7b97 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -119,7 +119,7 @@ def convert_infraction_name(cls, infr_type: str | Infraction) -> Infraction: async def action(self, ctx: FilterContext) -> None: """Send the notification to the user, and apply any specified infractions.""" # If there is no infraction to apply, any DM contents already provided in the context take precedence. - if self.infraction_type is None and (ctx.dm_content or ctx.dm_embed): + if self.infraction_type == Infraction.NONE and (ctx.dm_content or ctx.dm_embed): dm_content = ctx.dm_content dm_embed = ctx.dm_embed else: From 4d51cad0831f2a358d4674035bebfda2884d33ed Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 13:47:24 +0200 Subject: [PATCH 101/132] Deprecated `warn` -> `warning` Co-authored-by: wookie184 --- bot/exts/filtering/_filter_lists/antispam.py | 2 +- bot/exts/filtering/_filter_lists/filter_list.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index e549404c47..147998c1c2 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -49,7 +49,7 @@ def get_filter_type(self, content: str) -> type[UniqueFilter] | None: return antispam_filter_types[content] except KeyError: if content not in self._already_warned: - log.warn(f"An antispam filter named {content} was supplied, but no matching implementation found.") + log.warning(f"An antispam filter named {content} was supplied, but no matching implementation found.") self._already_warned.add(content) return None diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 4913315ebb..bf02071cfb 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -220,7 +220,7 @@ def _create_filter(self, filter_data: dict, defaults: Defaults) -> T | None: if filter_type: return filter_type(filter_data, defaults) elif content not in self._already_warned: - log.warn(f"A filter named {content} was supplied, but no matching implementation found.") + log.warning(f"A filter named {content} was supplied, but no matching implementation found.") self._already_warned.add(content) return None except TypeError as e: From 9b764e5a1c8b36ce16adec42e4913ae5b5f65499 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 13:51:18 +0200 Subject: [PATCH 102/132] Correct type hint Co-authored-by: wookie184 --- bot/utils/message_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/message_cache.py b/bot/utils/message_cache.py index 3e77e6a50e..5deb2376ba 100644 --- a/bot/utils/message_cache.py +++ b/bot/utils/message_cache.py @@ -105,7 +105,7 @@ def get_message(self, message_id: int) -> Message | None: index = self._message_id_mapping.get(message_id, None) return self._messages[index] if index is not None else None - def get_message_metadata(self, message_id: int) -> dict: + def get_message_metadata(self, message_id: int) -> dict | None: """Return the metadata of the message that has the given message ID, if it is cached.""" return self._message_metadata.get(message_id, None) From bd5c0f88851bfd074773d2ef69732e667a9e39d8 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 14:30:09 +0200 Subject: [PATCH 103/132] Re-add webhook and discord token check in other cogs Co-authored-by: wookie184 --- bot/exts/filtering/_filters/unique/discord_token.py | 2 +- bot/exts/info/codeblock/_cog.py | 4 ++++ bot/exts/moderation/watchchannels/_watchchannel.py | 6 +++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_filters/unique/discord_token.py b/bot/exts/filtering/_filters/unique/discord_token.py index 731df198c8..b2ac43cdbd 100644 --- a/bot/exts/filtering/_filters/unique/discord_token.py +++ b/bot/exts/filtering/_filters/unique/discord_token.py @@ -143,7 +143,7 @@ def format_log_message(cls, author: discord.User, channel: discord.abc.GuildChan @classmethod def find_token_in_message(cls, content: str) -> Token | None: - """Return a seemingly valid token found in `msg` or `None` if no token is found.""" + """Return a seemingly valid token found in `content` or `None` if no token is found.""" # Use finditer rather than search to guard against method calls prematurely returning the # token check (e.g. `message.channel.send` also matches our token pattern) for match in TOKEN_RE.finditer(content): diff --git a/bot/exts/info/codeblock/_cog.py b/bot/exts/info/codeblock/_cog.py index cc58621315..95ea547617 100644 --- a/bot/exts/info/codeblock/_cog.py +++ b/bot/exts/info/codeblock/_cog.py @@ -8,6 +8,8 @@ from bot import constants from bot.bot import Bot +from bot.exts.filtering._filters.unique.discord_token import DiscordTokenFilter +from bot.exts.filtering._filters.unique.webhook import WEBHOOK_URL_RE from bot.exts.info.codeblock._instructions import get_instructions from bot.log import get_logger from bot.utils import has_lines @@ -133,6 +135,8 @@ def should_parse(self, message: discord.Message) -> bool: not message.author.bot and self.is_valid_channel(message.channel) and has_lines(message.content, constants.CodeBlock.minimum_lines) + and not DiscordTokenFilter.find_token_in_message(message.content) + and not WEBHOOK_URL_RE.search(message.content) ) @Cog.listener() diff --git a/bot/exts/moderation/watchchannels/_watchchannel.py b/bot/exts/moderation/watchchannels/_watchchannel.py index dc73aeef00..8701320deb 100644 --- a/bot/exts/moderation/watchchannels/_watchchannel.py +++ b/bot/exts/moderation/watchchannels/_watchchannel.py @@ -14,6 +14,8 @@ from bot.bot import Bot from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons +from bot.exts.filtering._filters.unique.discord_token import DiscordTokenFilter +from bot.exts.filtering._filters.unique.webhook import WEBHOOK_URL_RE from bot.exts.moderation.modlog import ModLog from bot.log import CustomLogger, get_logger from bot.pagination import LinePaginator @@ -233,7 +235,9 @@ async def relay_message(self, msg: Message) -> None: await self.send_header(msg) - if cleaned_content := msg.clean_content: + if DiscordTokenFilter.find_token_in_message(msg.content) or WEBHOOK_URL_RE.search(msg.content): + cleaned_content = "Content is censored because it contains a bot or webhook token." + elif cleaned_content := msg.clean_content: # Put all non-media URLs in a code block to prevent embeds media_urls = {embed.url for embed in msg.embeds if embed.type in ("image", "video")} for url in URL_RE.findall(cleaned_content): From 93cc2596931a652ae46fb80cbeff5b991434d779 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 15:29:51 +0200 Subject: [PATCH 104/132] Don't allow adding filter lists with no implementation Filter lists with no implemenation are not loaded, therefore if one is added with no implementation there is then no way to interact with it. Instead of accounting for that case it makes more sense to require an implemenation first. --- bot/exts/filtering/filtering.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 66f4715f31..26e0f32d25 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -742,6 +742,15 @@ async def fl_describe( @has_any_role(Roles.admins) async def fl_add(self, ctx: Context, list_type: list_type_converter, list_name: str) -> None: """Add a new filter list.""" + # Check if there's an implementation. + if list_name.lower() not in filter_list_types: + if list_name.lower()[:-1] not in filter_list_types: # Maybe the name was given with uppercase or in plural? + await ctx.reply(f":x: Cannot add a `{list_name}` filter list, as there is no matching implementation.") + return + else: + list_name = list_name.lower()[:-1] + + # Check it doesn't already exist. list_description = f"{past_tense(list_type.name.lower())} {list_name.lower()}" if list_name in self.filter_lists: filter_list = self.filter_lists[list_name] From d4926d394fe14a666c238de6e35ef10cabb418e0 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 15:54:30 +0200 Subject: [PATCH 105/132] Copy message from other infraction if result doesn't have one. --- .../actions/infraction_and_notification.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 389d2b7b97..48249115b9 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -168,17 +168,20 @@ def union(self, other: Self) -> Self: There is no clear way to properly combine several notification messages, especially when it's in two parts. To avoid bombarding the user with several notifications, the message with the more significant infraction - is used. + is used. If the more significant infraction has no accompanying message, use the one from the other infraction, + if it exists. """ # Lower number -> higher in the hierarchy if self.infraction_type is None: return other.copy() elif other.infraction_type is None: return self.copy() - elif self.infraction_type.value < other.infraction_type.value: - return self.copy() + + if self.infraction_type.value < other.infraction_type.value: + result = self.copy() elif self.infraction_type.value > other.infraction_type.value: - return other.copy() + result = other.copy() + other = self else: if self.infraction_duration is None or ( other.infraction_duration is not None and self.infraction_duration > other.infraction_duration @@ -186,4 +189,15 @@ def union(self, other: Self) -> Self: result = self.copy() else: result = other.copy() - return result + other = self + + # If the winner has no message but the loser does, copy the message to the winner. + result_overrides = result.overrides + if "dm_content" not in result_overrides and "dm_embed" not in result_overrides: + other_overrides = other.overrides + if "dm_content" in other_overrides: + result.dm_content = other_overrides["dm_content"] + if "dm_embed" in other_overrides: + result.dm_content = other_overrides["dm_embed"] + + return result From 3211097574ffaee1026f200272f33cc0bbc73ed7 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Feb 2023 20:41:59 +0200 Subject: [PATCH 106/132] Fix antispam alerting - Only upload deletion logs when there's actually deletion. - Don't upload deletion logs when an antispam event begins (could happen when another filter demanded an alert to be sent), but instead delay it until the antispam alert. Otherwise the antispam alert would cause an error because it would try to upload duplicate messages. - Correctly include any messages and channels added to the antispam event after the initial deletion. --- bot/exts/filtering/_filter_context.py | 4 +++- bot/exts/filtering/_filter_lists/antispam.py | 18 +++++++++++++----- .../_settings_types/actions/remove_context.py | 2 ++ bot/exts/filtering/_ui/ui.py | 17 ++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 9faa16b533..f5f635f9c3 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -44,11 +44,13 @@ class FilterContext: matches: list[str] = field(default_factory=list) # What exactly was found notification_domain: str = "" # A domain to send the user for context filter_info: dict['Filter', str] = field(default_factory=dict) # Additional info from a filter. + messages_deletion: bool = False # Whether the messages were deleted. Can't upload deletion log otherwise. # Additional actions to perform additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list) - related_messages: set[Message] = field(default_factory=set) + related_messages: set[Message] = field(default_factory=set) # Deletion will include these. related_channels: set[TextChannel | Thread | DMChannel] = field(default_factory=set) attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. + upload_deletion_logs: bool = True # Whether it's allowed to upload deletion logs. @classmethod def from_message( diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index 147998c1c2..c5ce292e38 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -77,17 +77,23 @@ async def actions_for( self.message_deletion_queue[ctx.author] = DeletionContext() ctx.additional_actions.append(self._create_deletion_context_handler(ctx.author)) ctx.related_channels |= {msg.channel for msg in ctx.related_messages} - else: # The additional messages found are already part of the deletion context + else: # The additional messages found are already part of a deletion context ctx.related_messages = set() current_infraction = self.message_deletion_queue[ctx.author].current_infraction + # In case another filter wants an alert, prevent deleted messages from being uploaded now and also for + # the spam alert (upload happens during alerting). + # Deleted messages API doesn't accept duplicates and will error. + # Additional messages are necessarily part of the deletion. + ctx.upload_deletion_logs = False self.message_deletion_queue[ctx.author].add(ctx, triggers) - current_actions = sublist.merge_actions(triggers) if triggers else None + current_actions = sublist.merge_actions(triggers) # Don't alert yet. current_actions.pop("ping", None) current_actions.pop("send_alert", None) + new_infraction = current_actions["infraction_and_notification"].copy() - # Smaller infraction value = higher in hierarchy. + # Smaller infraction value => higher in hierarchy. if not current_infraction or new_infraction.infraction_type.value < current_infraction.value: # Pick the first triggered filter for the reason, there's no good way to decide between them. new_infraction.infraction_reason = ( @@ -161,11 +167,13 @@ async def send_alert(self, antispam_list: AntispamList) -> None: new_ctx.action_descriptions[-1] += f" (+{descriptions_num - 20} other actions)" new_ctx.related_messages = reduce( or_, (other_ctx.related_messages for other_ctx in other_contexts), ctx.related_messages - ) + ) | {ctx.message for ctx in other_contexts} new_ctx.related_channels = reduce( or_, (other_ctx.related_channels for other_ctx in other_contexts), ctx.related_channels - ) + ) | {ctx.channel for ctx in other_contexts} new_ctx.attachments = reduce(or_, (other_ctx.attachments for other_ctx in other_contexts), ctx.attachments) + new_ctx.upload_deletion_logs = True + new_ctx.messages_deletion = all(ctx.messages_deletion for ctx in self.contexts) rules = list(self.rules) actions = antispam_list[ListType.DENY].merge_actions(rules) diff --git a/bot/exts/filtering/_settings_types/actions/remove_context.py b/bot/exts/filtering/_settings_types/actions/remove_context.py index e030f06d2b..a348225429 100644 --- a/bot/exts/filtering/_settings_types/actions/remove_context.py +++ b/bot/exts/filtering/_settings_types/actions/remove_context.py @@ -59,6 +59,8 @@ async def _handle_messages(ctx: FilterContext) -> None: if not ctx.message or not ctx.message.guild: return + # If deletion somehow fails at least this will allow scheduling for deletion. + ctx.messages_deletion = True channel_messages = defaultdict(set) # Duplicates will cause batch deletion to fail. for message in {ctx.message} | ctx.related_messages: channel_messages[message.channel].add(message) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index dc996faf72..24fb507e3d 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -57,22 +57,25 @@ async def _build_alert_message_content(ctx: FilterContext, current_message_length: int) -> str: """Build the content section of the alert.""" # For multiple messages and those with attachments or excessive newlines, use the logs API - if any(( + if ctx.messages_deletion and ctx.upload_deletion_logs and any(( ctx.related_messages, len(ctx.attachments) > 0, ctx.content.count('\n') > 15 )): url = await upload_log(ctx.related_messages, bot.instance.user.id, ctx.attachments) - alert_content = f"A complete log of the offending messages can be found [here]({url})" - else: - alert_content = escape_markdown(ctx.content) - remaining_chars = MAX_EMBED_DESCRIPTION - current_message_length + return f"A complete log of the offending messages can be found [here]({url})" + + alert_content = escape_markdown(ctx.content) + remaining_chars = MAX_EMBED_DESCRIPTION - current_message_length - if len(alert_content) > remaining_chars: + if len(alert_content) > remaining_chars: + if ctx.messages_deletion and ctx.upload_deletion_logs: url = await upload_log([ctx.message], bot.instance.user.id, ctx.attachments) log_site_msg = f"The full message can be found [here]({url})" # 7 because that's the length of "[...]\n\n" - alert_content = alert_content[:remaining_chars - (7 + len(log_site_msg))] + "[...]\n\n" + log_site_msg + return alert_content[:remaining_chars - (7 + len(log_site_msg))] + "[...]\n\n" + log_site_msg + else: + return alert_content[:remaining_chars - 5] + "[...]" return alert_content From fe41b683cc9e5d87e1490c09ebfbbc79d5c8f9a9 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 4 Mar 2023 12:39:59 +0200 Subject: [PATCH 107/132] Remove empty README --- bot/exts/filtering/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 bot/exts/filtering/README.md diff --git a/bot/exts/filtering/README.md b/bot/exts/filtering/README.md deleted file mode 100644 index e69de29bb2..0000000000 From 659320edd6609c4a287f5da1ff1ce6436c4b6605 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 4 Mar 2023 13:10:13 +0200 Subject: [PATCH 108/132] More type hint parameterizing --- bot/exts/filtering/_filter_context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index f5f635f9c3..8e1ed5788a 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -29,7 +29,7 @@ class FilterContext: event: Event # The type of event author: User | Member | None # Who triggered the event channel: TextChannel | Thread | DMChannel | None # The channel involved - content: str | Iterable # What actually needs filtering + content: str | Iterable # What actually needs filtering. The Iterable type depends on the filter list. message: Message | None # The message involved embeds: list[Embed] = field(default_factory=list) # Any embeds involved before_message: Message | None = None @@ -40,7 +40,7 @@ class FilterContext: send_alert: bool = False # Whether to send an alert for the moderators alert_content: str = "" # The content of the alert alert_embeds: list[Embed] = field(default_factory=list) # Any embeds to add to the alert - action_descriptions: list = field(default_factory=list) # What actions were taken + action_descriptions: list[str] = field(default_factory=list) # What actions were taken matches: list[str] = field(default_factory=list) # What exactly was found notification_domain: str = "" # A domain to send the user for context filter_info: dict['Filter', str] = field(default_factory=dict) # Additional info from a filter. From b64eee91c0e212e53307fbcdd72fe8794dd48d9d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 23 Mar 2023 19:51:06 +0200 Subject: [PATCH 109/132] Fix filtering tests --- tests/bot/exts/filtering/test_settings_entries.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py index 5a1eb6fe63..c5f0152b0f 100644 --- a/tests/bot/exts/filtering/test_settings_entries.py +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -152,7 +152,7 @@ def test_filtering_dms_when_necessary(self): def test_infraction_merge_of_same_infraction_type(self): """When both infractions are of the same type, the one with the longer duration wins.""" infraction1 = InfractionAndNotification( - infraction_type="MUTE", + infraction_type="TIMEOUT", infraction_reason="hi", infraction_duration=10, dm_content="how", @@ -160,7 +160,7 @@ def test_infraction_merge_of_same_infraction_type(self): infraction_channel=0 ) infraction2 = InfractionAndNotification( - infraction_type="MUTE", + infraction_type="TIMEOUT", infraction_reason="there", infraction_duration=20, dm_content="are you", @@ -168,12 +168,12 @@ def test_infraction_merge_of_same_infraction_type(self): infraction_channel=0 ) - result = infraction1 | infraction2 + result = infraction1.union(infraction2) self.assertDictEqual( result.dict(), { - "infraction_type": Infraction.MUTE, + "infraction_type": Infraction.TIMEOUT, "infraction_reason": "there", "infraction_duration": 20.0, "dm_content": "are you", @@ -185,7 +185,7 @@ def test_infraction_merge_of_same_infraction_type(self): def test_infraction_merge_of_different_infraction_types(self): """If there are two different infraction types, the one higher up the hierarchy should be picked.""" infraction1 = InfractionAndNotification( - infraction_type="MUTE", + infraction_type="TIMEOUT", infraction_reason="hi", infraction_duration=20, dm_content="", @@ -201,7 +201,7 @@ def test_infraction_merge_of_different_infraction_types(self): infraction_channel=0 ) - result = infraction1 | infraction2 + result = infraction1.union(infraction2) self.assertDictEqual( result.dict(), From 509c7968dab875f8e3e7934647c757a2a73f724b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Thu, 23 Mar 2023 19:59:05 +0200 Subject: [PATCH 110/132] Add support for snekbox IO in the new filtering system --- bot/exts/filtering/_filter_context.py | 19 ++++- bot/exts/filtering/_filter_lists/antispam.py | 4 +- bot/exts/filtering/_filter_lists/domain.py | 2 +- bot/exts/filtering/_filter_lists/extension.py | 56 +++++++------ bot/exts/filtering/_filter_lists/invite.py | 2 +- bot/exts/filtering/_filter_lists/token.py | 2 +- .../_filters/unique/discord_token.py | 2 +- .../filtering/_filters/unique/everyone.py | 2 +- bot/exts/filtering/_filters/unique/webhook.py | 2 +- .../_settings_types/actions/remove_context.py | 4 +- bot/exts/filtering/_ui/ui.py | 6 +- bot/exts/filtering/filtering.py | 17 ++-- bot/exts/utils/snekbox/_cog.py | 84 ++++++++----------- bot/exts/utils/snekbox/_io.py | 10 +-- .../exts/filtering/test_extension_filter.py | 30 +++---- tests/bot/exts/utils/snekbox/test_snekbox.py | 8 +- 16 files changed, 134 insertions(+), 116 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 8e1ed5788a..483706e2af 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -5,12 +5,14 @@ from dataclasses import dataclass, field, replace from enum import Enum, auto +import discord from discord import DMChannel, Embed, Member, Message, TextChannel, Thread, User from bot.utils.message_cache import MessageCache if typing.TYPE_CHECKING: from bot.exts.filtering._filters.filter import Filter + from bot.exts.utils.snekbox._io import FileAttachment class Event(Enum): @@ -19,6 +21,7 @@ class Event(Enum): MESSAGE = auto() MESSAGE_EDIT = auto() NICKNAME = auto() + SNEKBOX = auto() @dataclass @@ -32,6 +35,7 @@ class FilterContext: content: str | Iterable # What actually needs filtering. The Iterable type depends on the filter list. message: Message | None # The message involved embeds: list[Embed] = field(default_factory=list) # Any embeds involved + attachments: list[discord.Attachment | FileAttachment] = field(default_factory=list) # Any attachments sent. before_message: Message | None = None message_cache: MessageCache | None = None # Output context @@ -45,11 +49,12 @@ class FilterContext: notification_domain: str = "" # A domain to send the user for context filter_info: dict['Filter', str] = field(default_factory=dict) # Additional info from a filter. messages_deletion: bool = False # Whether the messages were deleted. Can't upload deletion log otherwise. + blocked_exts: set[str] = field(default_factory=set) # Any extensions blocked (used for snekbox) # Additional actions to perform additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list) related_messages: set[Message] = field(default_factory=set) # Deletion will include these. related_channels: set[TextChannel | Thread | DMChannel] = field(default_factory=set) - attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. + uploaded_attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. upload_deletion_logs: bool = True # Whether it's allowed to upload deletion logs. @classmethod @@ -57,7 +62,17 @@ def from_message( cls, event: Event, message: Message, before: Message | None = None, cache: MessageCache | None = None ) -> FilterContext: """Create a filtering context from the attributes of a message.""" - return cls(event, message.author, message.channel, message.content, message, message.embeds, before, cache) + return cls( + event, + message.author, + message.channel, + message.content, + message, + message.embeds, + message.attachments, + before, + cache + ) def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index 0e7ab2bdcd..ba20051fc9 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -171,7 +171,9 @@ async def send_alert(self, antispam_list: AntispamList) -> None: new_ctx.related_channels = reduce( or_, (other_ctx.related_channels for other_ctx in other_contexts), ctx.related_channels ) | {ctx.channel for ctx in other_contexts} - new_ctx.attachments = reduce(or_, (other_ctx.attachments for other_ctx in other_contexts), ctx.attachments) + new_ctx.uploaded_attachments = reduce( + or_, (other_ctx.uploaded_attachments for other_ctx in other_contexts), ctx.uploaded_attachments + ) new_ctx.upload_deletion_logs = True new_ctx.messages_deletion = all(ctx.messages_deletion for ctx in self.contexts) diff --git a/bot/exts/filtering/_filter_lists/domain.py b/bot/exts/filtering/_filter_lists/domain.py index f4062edfe4..091fd14e0c 100644 --- a/bot/exts/filtering/_filter_lists/domain.py +++ b/bot/exts/filtering/_filter_lists/domain.py @@ -31,7 +31,7 @@ class DomainsList(FilterList[DomainFilter]): def __init__(self, filtering_cog: Filtering): super().__init__() - filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT, Event.SNEKBOX) def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index a739d7191b..868fde2b2b 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -49,7 +49,7 @@ class ExtensionsList(FilterList[ExtensionFilter]): def __init__(self, filtering_cog: Filtering): super().__init__() - filtering_cog.subscribe(self, Event.MESSAGE) + filtering_cog.subscribe(self, Event.MESSAGE, Event.SNEKBOX) self._whitelisted_description = None def get_filter_type(self, content: str) -> type[Filter]: @@ -66,7 +66,7 @@ async def actions_for( ) -> tuple[ActionSettings | None, list[str], dict[ListType, list[Filter]]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" # Return early if the message doesn't have attachments. - if not ctx.message or not ctx.message.attachments: + if not ctx.message or not ctx.attachments: return None, [], {} _, failed = self[ListType.ALLOW].defaults.validations.evaluate(ctx) @@ -75,7 +75,7 @@ async def actions_for( # Find all extensions in the message. all_ext = { - (splitext(attachment.filename.lower())[1], attachment.filename) for attachment in ctx.message.attachments + (splitext(attachment.filename.lower())[1], attachment.filename) for attachment in ctx.attachments } new_ctx = ctx.replace(content={ext for ext, _ in all_ext}) # And prepare the context for the filters to read. triggered = [ @@ -86,31 +86,37 @@ async def actions_for( # See if there are any extensions left which aren't allowed. not_allowed = {ext: filename for ext, filename in all_ext if ext not in allowed_ext} + if ctx.event == Event.SNEKBOX: + not_allowed = {ext: filename for ext, filename in not_allowed.items() if ext not in TXT_LIKE_FILES} + if not not_allowed: # Yes, it's a double negative. Meaning all attachments are allowed :) return None, [], {ListType.ALLOW: triggered} - # Something is disallowed. - if ".py" in not_allowed: - # Provide a pastebin link for .py files. - ctx.dm_embed = PY_EMBED_DESCRIPTION - elif txt_extensions := {ext for ext in TXT_LIKE_FILES if ext in not_allowed}: - # Work around Discord auto-conversion of messages longer than 2000 chars to .txt - cmd_channel = bot.instance.get_channel(Channels.bot_commands) - ctx.dm_embed = TXT_EMBED_DESCRIPTION.format( - blocked_extension=txt_extensions.pop(), - cmd_channel_mention=cmd_channel.mention - ) - else: - meta_channel = bot.instance.get_channel(Channels.meta) - if not self._whitelisted_description: - self._whitelisted_description = ', '.join( - filter_.content for filter_ in self[ListType.ALLOW].filters.values() + # At this point, something is disallowed. + if ctx.event != Event.SNEKBOX: # Don't post the embed if it's a snekbox response. + if ".py" in not_allowed: + # Provide a pastebin link for .py files. + ctx.dm_embed = PY_EMBED_DESCRIPTION + elif txt_extensions := {ext for ext in TXT_LIKE_FILES if ext in not_allowed}: + # Work around Discord auto-conversion of messages longer than 2000 chars to .txt + cmd_channel = bot.instance.get_channel(Channels.bot_commands) + ctx.dm_embed = TXT_EMBED_DESCRIPTION.format( + blocked_extension=txt_extensions.pop(), + cmd_channel_mention=cmd_channel.mention + ) + else: + meta_channel = bot.instance.get_channel(Channels.meta) + if not self._whitelisted_description: + self._whitelisted_description = ', '.join( + filter_.content for filter_ in self[ListType.ALLOW].filters.values() + ) + ctx.dm_embed = DISALLOWED_EMBED_DESCRIPTION.format( + joined_whitelist=self._whitelisted_description, + blocked_extensions_str=", ".join(not_allowed), + meta_channel_mention=meta_channel.mention, ) - ctx.dm_embed = DISALLOWED_EMBED_DESCRIPTION.format( - joined_whitelist=self._whitelisted_description, - blocked_extensions_str=", ".join(not_allowed), - meta_channel_mention=meta_channel.mention, - ) ctx.matches += not_allowed.values() - return self[ListType.ALLOW].defaults.actions, [f"`{ext}`" for ext in not_allowed], {ListType.ALLOW: triggered} + ctx.blocked_exts |= set(not_allowed) + actions = self[ListType.ALLOW].defaults.actions if ctx.event != Event.SNEKBOX else None + return actions, [f"`{ext}`" for ext in not_allowed], {ListType.ALLOW: triggered} diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index bd0eaa1221..b9732a6dc9 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -37,7 +37,7 @@ class InviteList(FilterList[InviteFilter]): def __init__(self, filtering_cog: Filtering): super().__init__() - filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT) + filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT, Event.SNEKBOX) def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" diff --git a/bot/exts/filtering/_filter_lists/token.py b/bot/exts/filtering/_filter_lists/token.py index f5da28bb5c..0c591ac3bb 100644 --- a/bot/exts/filtering/_filter_lists/token.py +++ b/bot/exts/filtering/_filter_lists/token.py @@ -32,7 +32,7 @@ class TokensList(FilterList[TokenFilter]): def __init__(self, filtering_cog: Filtering): super().__init__() - filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT, Event.NICKNAME) + filtering_cog.subscribe(self, Event.MESSAGE, Event.MESSAGE_EDIT, Event.NICKNAME, Event.SNEKBOX) def get_filter_type(self, content: str) -> type[Filter]: """Get a subclass of filter matching the filter list and the filter's content.""" diff --git a/bot/exts/filtering/_filters/unique/discord_token.py b/bot/exts/filtering/_filters/unique/discord_token.py index 6174ee30bd..f4b9cc7410 100644 --- a/bot/exts/filtering/_filters/unique/discord_token.py +++ b/bot/exts/filtering/_filters/unique/discord_token.py @@ -61,7 +61,7 @@ class DiscordTokenFilter(UniqueFilter): """Scans messages for potential discord client tokens and removes them.""" name = "discord_token" - events = (Event.MESSAGE, Event.MESSAGE_EDIT) + events = (Event.MESSAGE, Event.MESSAGE_EDIT, Event.SNEKBOX) extra_fields_type = ExtraDiscordTokenSettings @property diff --git a/bot/exts/filtering/_filters/unique/everyone.py b/bot/exts/filtering/_filters/unique/everyone.py index a32e67cc57..e49ede82fd 100644 --- a/bot/exts/filtering/_filters/unique/everyone.py +++ b/bot/exts/filtering/_filters/unique/everyone.py @@ -16,7 +16,7 @@ class EveryoneFilter(UniqueFilter): """Filter messages which contain `@everyone` and `@here` tags outside a codeblock.""" name = "everyone" - events = (Event.MESSAGE, Event.MESSAGE_EDIT) + events = (Event.MESSAGE, Event.MESSAGE_EDIT, Event.SNEKBOX) async def triggered_on(self, ctx: FilterContext) -> bool: """Search for the filter's content within a given context.""" diff --git a/bot/exts/filtering/_filters/unique/webhook.py b/bot/exts/filtering/_filters/unique/webhook.py index 965ef42eb0..4e1e2e44df 100644 --- a/bot/exts/filtering/_filters/unique/webhook.py +++ b/bot/exts/filtering/_filters/unique/webhook.py @@ -22,7 +22,7 @@ class WebhookFilter(UniqueFilter): """Scan messages to detect Discord webhooks links.""" name = "webhook" - events = (Event.MESSAGE, Event.MESSAGE_EDIT) + events = (Event.MESSAGE, Event.MESSAGE_EDIT, Event.SNEKBOX) @property def mod_log(self) -> ModLog | None: diff --git a/bot/exts/filtering/_settings_types/actions/remove_context.py b/bot/exts/filtering/_settings_types/actions/remove_context.py index 7ead888187..5ec2613f41 100644 --- a/bot/exts/filtering/_settings_types/actions/remove_context.py +++ b/bot/exts/filtering/_settings_types/actions/remove_context.py @@ -28,8 +28,8 @@ async def upload_messages_attachments(ctx: FilterContext, messages: list[Message return destination = messages[0].guild.get_channel(Channels.attachment_log) for message in messages: - if message.attachments and message.id not in ctx.attachments: - ctx.attachments[message.id] = await send_attachments(message, destination, link_large=False) + if message.attachments and message.id not in ctx.uploaded_attachments: + ctx.uploaded_attachments[message.id] = await send_attachments(message, destination, link_large=False) class RemoveContext(ActionEntry): diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 157906d6bb..8cd2864a93 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -59,10 +59,10 @@ async def _build_alert_message_content(ctx: FilterContext, current_message_lengt # For multiple messages and those with attachments or excessive newlines, use the logs API if ctx.messages_deletion and ctx.upload_deletion_logs and any(( ctx.related_messages, - len(ctx.attachments) > 0, + len(ctx.uploaded_attachments) > 0, ctx.content.count('\n') > 15 )): - url = await upload_log(ctx.related_messages, bot.instance.user.id, ctx.attachments) + url = await upload_log(ctx.related_messages, bot.instance.user.id, ctx.uploaded_attachments) return f"A complete log of the offending messages can be found [here]({url})" alert_content = escape_markdown(ctx.content) @@ -70,7 +70,7 @@ async def _build_alert_message_content(ctx: FilterContext, current_message_lengt if len(alert_content) > remaining_chars: if ctx.messages_deletion and ctx.upload_deletion_logs: - url = await upload_log([ctx.message], bot.instance.user.id, ctx.attachments) + url = await upload_log([ctx.message], bot.instance.user.id, ctx.uploaded_attachments) log_site_msg = f"The full message can be found [here]({url})" # 7 because that's the length of "[...]\n\n" return alert_content[:remaining_chars - (7 + len(log_site_msg))] + "[...]\n\n" + log_site_msg diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index c4417e5e06..2a7f8f81f0 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -40,6 +40,7 @@ ) from bot.exts.filtering._utils import past_tense, repr_equals, starting_value, to_serializable from bot.exts.moderation.infraction.infractions import COMP_BAN_DURATION, COMP_BAN_REASON +from bot.exts.utils.snekbox._io import FileAttachment from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.channel import is_mod_channel @@ -251,24 +252,30 @@ async def on_voice_state_update(self, member: discord.Member, *_) -> None: ctx = FilterContext(Event.NICKNAME, member, None, member.display_name, None) await self._check_bad_name(ctx) - async def filter_snekbox_output(self, snekbox_result: str, msg: Message) -> bool: + async def filter_snekbox_output( + self, stdout: str, files: list[FileAttachment], msg: Message + ) -> tuple[bool, set[str]]: """ Filter the result of a snekbox command to see if it violates any of our rules, and then respond accordingly. Also requires the original message, to check whether to filter and for alerting. Any action (deletion, infraction) will be applied in the context of the original message. - Returns whether a filter was triggered or not. + Returns whether the output should be blocked, as well as a list of blocked file extensions. """ - ctx = FilterContext.from_message(Event.MESSAGE, msg).replace(content=snekbox_result) + content = stdout + if files: # Filter the filenames as well. + content += "\n\n" + "\n".join(file.filename for file in files) + ctx = FilterContext.from_message(Event.SNEKBOX, msg).replace(content=content, attachments=files) + result_actions, list_messages, triggers = await self._resolve_action(ctx) if result_actions: await result_actions.action(ctx) if ctx.send_alert: await self._send_alert(ctx, list_messages) - self._increment_stats(triggers) - return result_actions is not None + self._increment_stats(triggers) + return result_actions is not None, ctx.blocked_exts # endregion # region: blacklist commands diff --git a/bot/exts/utils/snekbox/_cog.py b/bot/exts/utils/snekbox/_cog.py index 567fe6c243..d7e8bc93c7 100644 --- a/bot/exts/utils/snekbox/_cog.py +++ b/bot/exts/utils/snekbox/_cog.py @@ -14,9 +14,8 @@ from pydis_core.utils.regex import FORMATTED_CODE_REGEX, RAW_CODE_REGEX from bot.bot import Bot -from bot.constants import Channels, Emojis, MODERATION_ROLES, Roles, STAFF_PARTNERS_COMMUNITY_ROLES, URLs +from bot.constants import Channels, Emojis, MODERATION_ROLES, Roles, URLs from bot.decorators import redirect_output -from bot.exts.events.code_jams._channels import CATEGORY_NAME as JAM_CATEGORY_NAME from bot.exts.filtering._filter_lists.extension import TXT_LIKE_FILES from bot.exts.help_channels._channel import is_help_forum_post from bot.exts.utils.snekbox._eval import EvalJob, EvalResult @@ -288,37 +287,22 @@ async def format_output( return output, paste_link - def get_extensions_whitelist(self) -> set[str]: - """Return a set of whitelisted file extensions.""" - return set(self.bot.filter_list_cache['FILE_FORMAT.True'].keys()) | TXT_LIKE_FILES - - def _filter_files(self, ctx: Context, files: list[FileAttachment]) -> FilteredFiles: + def _filter_files(self, ctx: Context, files: list[FileAttachment], blocked_exts: set[str]) -> FilteredFiles: """Filter to restrict files to allowed extensions. Return a named tuple of allowed and blocked files lists.""" - # Check if user is staff, if is, return - # Since we only care that roles exist to iterate over, check for the attr rather than a User/Member instance - if hasattr(ctx.author, "roles") and any(role.id in STAFF_PARTNERS_COMMUNITY_ROLES for role in ctx.author.roles): - return FilteredFiles(files, []) - # Ignore code jam channels - if getattr(ctx.channel, "category", None) and ctx.channel.category.name == JAM_CATEGORY_NAME: - return FilteredFiles(files, []) - - # Get whitelisted extensions - whitelist = self.get_extensions_whitelist() - # Filter files into allowed and blocked blocked = [] allowed = [] for file in files: - if file.suffix in whitelist: - allowed.append(file) - else: + if file.suffix in blocked_exts: blocked.append(file) + else: + allowed.append(file) if blocked: blocked_str = ", ".join(f.suffix for f in blocked) log.info( f"User '{ctx.author}' ({ctx.author.id}) uploaded blacklisted file(s) in eval: {blocked_str}", - extra={"attachment_list": [f.path for f in files]} + extra={"attachment_list": [f.filename for f in files]} ) return FilteredFiles(allowed, blocked) @@ -365,31 +349,8 @@ async def send_job(self, ctx: Context, job: EvalJob) -> Message: else: self.bot.stats.incr("snekbox.python.success") - # Filter file extensions - allowed, blocked = self._filter_files(ctx, result.files) - # Also scan failed files for blocked extensions - failed_files = [FileAttachment(name, b"") for name in result.failed_files] - blocked.extend(self._filter_files(ctx, failed_files).blocked) - # Add notice if any files were blocked - if blocked: - blocked_sorted = sorted(set(f.suffix for f in blocked)) - # Only no extension - if len(blocked_sorted) == 1 and blocked_sorted[0] == "": - blocked_msg = "Files with no extension can't be uploaded." - # Both - elif "" in blocked_sorted: - blocked_str = ", ".join(ext for ext in blocked_sorted if ext) - blocked_msg = ( - f"Files with no extension or disallowed extensions can't be uploaded: **{blocked_str}**" - ) - else: - blocked_str = ", ".join(blocked_sorted) - blocked_msg = f"Files with disallowed extensions can't be uploaded: **{blocked_str}**" - - msg += f"\n{Emojis.failed_file} {blocked_msg}" - # Split text files - text_files = [f for f in allowed if f.suffix in TXT_LIKE_FILES] + text_files = [f for f in result.files if f.suffix in TXT_LIKE_FILES] # Inline until budget, then upload to paste service # Budget is shared with stdout, so subtract what we've already used budget_lines = MAX_OUTPUT_BLOCK_LINES - (output.count("\n") + 1) @@ -417,8 +378,35 @@ async def send_job(self, ctx: Context, job: EvalJob) -> Message: budget_chars -= len(file_text) filter_cog: Filtering | None = self.bot.get_cog("Filtering") - if filter_cog and (await filter_cog.filter_snekbox_output(msg, ctx.message)): - return await ctx.send("Attempt to circumvent filter detected. Moderator team has been alerted.") + blocked_exts = set() + # Include failed files in the scan. + failed_files = [FileAttachment(name, b"") for name in result.failed_files] + total_files = result.files + failed_files + if filter_cog: + block_output, blocked_exts = await filter_cog.filter_snekbox_output(msg, total_files, ctx.message) + if block_output: + return await ctx.send("Attempt to circumvent filter detected. Moderator team has been alerted.") + + # Filter file extensions + allowed, blocked = self._filter_files(ctx, result.files, blocked_exts) + blocked.extend(self._filter_files(ctx, failed_files, blocked_exts).blocked) + # Add notice if any files were blocked + if blocked: + blocked_sorted = sorted(set(f.suffix for f in blocked)) + # Only no extension + if len(blocked_sorted) == 1 and blocked_sorted[0] == "": + blocked_msg = "Files with no extension can't be uploaded." + # Both + elif "" in blocked_sorted: + blocked_str = ", ".join(ext for ext in blocked_sorted if ext) + blocked_msg = ( + f"Files with no extension or disallowed extensions can't be uploaded: **{blocked_str}**" + ) + else: + blocked_str = ", ".join(blocked_sorted) + blocked_msg = f"Files with disallowed extensions can't be uploaded: **{blocked_str}**" + + msg += f"\n{Emojis.failed_file} {blocked_msg}" # Upload remaining non-text files files = [f.to_file() for f in allowed if f not in text_files] diff --git a/bot/exts/utils/snekbox/_io.py b/bot/exts/utils/snekbox/_io.py index 9be3963355..a45ecec1a3 100644 --- a/bot/exts/utils/snekbox/_io.py +++ b/bot/exts/utils/snekbox/_io.py @@ -53,23 +53,23 @@ def normalize_discord_file_name(name: str) -> str: class FileAttachment: """File Attachment from Snekbox eval.""" - path: str + filename: str content: bytes def __repr__(self) -> str: """Return the content as a string.""" content = f"{self.content[:10]}..." if len(self.content) > 10 else self.content - return f"FileAttachment(path={self.path!r}, content={content})" + return f"FileAttachment(path={self.filename!r}, content={content})" @property def suffix(self) -> str: """Return the file suffix.""" - return PurePosixPath(self.path).suffix + return PurePosixPath(self.filename).suffix @property def name(self) -> str: """Return the file name.""" - return PurePosixPath(self.path).name + return PurePosixPath(self.filename).name @classmethod def from_dict(cls, data: dict, size_limit: int = FILE_SIZE_LIMIT) -> FileAttachment: @@ -92,7 +92,7 @@ def to_dict(self) -> dict[str, str]: content = content.encode("utf-8") return { - "path": self.path, + "path": self.filename, "content": b64encode(content).decode("ascii"), } diff --git a/tests/bot/exts/filtering/test_extension_filter.py b/tests/bot/exts/filtering/test_extension_filter.py index 0ad41116d4..351daa0b4c 100644 --- a/tests/bot/exts/filtering/test_extension_filter.py +++ b/tests/bot/exts/filtering/test_extension_filter.py @@ -45,9 +45,9 @@ def setUp(self): async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should trigger the whitelist and result in no actions or messages.""" attachment = MockAttachment(filename="python.first") - self.message.attachments = [attachment] + ctx = self.ctx.replace(attachments=[attachment]) - result = await self.filter_list.actions_for(self.ctx) + result = await self.filter_list.actions_for(ctx) self.assertEqual(result, (None, [], {ListType.ALLOW: [self.filter_list[ListType.ALLOW].filters[1]]})) @@ -62,9 +62,9 @@ async def test_message_without_attachment(self): async def test_message_with_illegal_extension(self): """A message with an illegal extension shouldn't trigger the whitelist, and return some action and message.""" attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] + ctx = self.ctx.replace(attachments=[attachment]) - result = await self.filter_list.actions_for(self.ctx) + result = await self.filter_list.actions_for(ctx) self.assertEqual(result, ({}, ["`.disallowed`"], {ListType.ALLOW: []})) @@ -72,11 +72,11 @@ async def test_message_with_illegal_extension(self): async def test_python_file_redirect_embed_description(self): """A message containing a .py file should result in an embed redirecting the user to our paste site.""" attachment = MockAttachment(filename="python.py") - self.message.attachments = [attachment] + ctx = self.ctx.replace(attachments=[attachment]) - await self.filter_list.actions_for(self.ctx) + await self.filter_list.actions_for(ctx) - self.assertEqual(self.ctx.dm_embed, extension.PY_EMBED_DESCRIPTION) + self.assertEqual(ctx.dm_embed, extension.PY_EMBED_DESCRIPTION) @patch("bot.instance", BOT) async def test_txt_file_redirect_embed_description(self): @@ -91,12 +91,12 @@ async def test_txt_file_redirect_embed_description(self): with self.subTest(file_name=file_name, disallowed_extension=disallowed_extension): attachment = MockAttachment(filename=f"{file_name}{disallowed_extension}") - self.message.attachments = [attachment] + ctx = self.ctx.replace(attachments=[attachment]) - await self.filter_list.actions_for(self.ctx) + await self.filter_list.actions_for(ctx) self.assertEqual( - self.ctx.dm_embed, + ctx.dm_embed, extension.TXT_EMBED_DESCRIPTION.format( blocked_extension=disallowed_extension, ) @@ -106,13 +106,13 @@ async def test_txt_file_redirect_embed_description(self): async def test_other_disallowed_extension_embed_description(self): """Test the description for a non .py/.txt/.json/.csv disallowed extension.""" attachment = MockAttachment(filename="python.disallowed") - self.message.attachments = [attachment] + ctx = self.ctx.replace(attachments=[attachment]) - await self.filter_list.actions_for(self.ctx) + await self.filter_list.actions_for(ctx) meta_channel = BOT.get_channel(Channels.meta) self.assertEqual( - self.ctx.dm_embed, + ctx.dm_embed, extension.DISALLOWED_EMBED_DESCRIPTION.format( joined_whitelist=", ".join(self.whitelist), blocked_extensions_str=".disallowed", @@ -134,6 +134,6 @@ async def test_get_disallowed_extensions(self): for extensions, expected_disallowed_extensions in test_values: with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): - self.message.attachments = [MockAttachment(filename=f"filename{ext}") for ext in extensions] - result = await self.filter_list.actions_for(self.ctx) + ctx = self.ctx.replace(attachments=[MockAttachment(filename=f"filename{ext}") for ext in extensions]) + result = await self.filter_list.actions_for(ctx) self.assertCountEqual(result[1], expected_disallowed_extensions) diff --git a/tests/bot/exts/utils/snekbox/test_snekbox.py b/tests/bot/exts/utils/snekbox/test_snekbox.py index 9dcf7fd8c0..79ac8ea2c3 100644 --- a/tests/bot/exts/utils/snekbox/test_snekbox.py +++ b/tests/bot/exts/utils/snekbox/test_snekbox.py @@ -307,7 +307,7 @@ async def test_send_job(self): self.cog.upload_output = AsyncMock() # Should not be called mocked_filter_cog = MagicMock() - mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=False) + mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=(False, [])) self.bot.get_cog.return_value = mocked_filter_cog job = EvalJob.from_code('MyAwesomeCode') @@ -339,7 +339,7 @@ async def test_send_job_with_paste_link(self): self.cog.format_output = AsyncMock(return_value=('Way too long beard', 'lookatmybeard.com')) mocked_filter_cog = MagicMock() - mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=False) + mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=(False, [])) self.bot.get_cog.return_value = mocked_filter_cog job = EvalJob.from_code("MyAwesomeCode").as_version("3.11") @@ -368,7 +368,7 @@ async def test_send_job_with_non_zero_eval(self): self.cog.upload_output = AsyncMock() # This function isn't called mocked_filter_cog = MagicMock() - mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=False) + mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=(False, [])) self.bot.get_cog.return_value = mocked_filter_cog job = EvalJob.from_code("MyAwesomeCode").as_version("3.11") @@ -396,7 +396,7 @@ async def test_send_job_with_disallowed_file_ext(self): self.cog.upload_output = AsyncMock() # This function isn't called mocked_filter_cog = MagicMock() - mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=False) + mocked_filter_cog.filter_snekbox_output = AsyncMock(return_value=(False, [".disallowed"])) self.bot.get_cog.return_value = mocked_filter_cog job = EvalJob.from_code("MyAwesomeCode").as_version("3.11") From caf6bd4377e3f5a6426bc32df4bf711a897b62b1 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 26 Mar 2023 23:13:50 +0300 Subject: [PATCH 111/132] Rename additional_field to additional_settings --- bot/exts/filtering/_filters/filter.py | 4 +++- bot/exts/filtering/filtering.py | 4 ++-- tests/bot/exts/filtering/test_discord_token_filter.py | 2 +- tests/bot/exts/filtering/test_extension_filter.py | 2 +- tests/bot/exts/filtering/test_token_filter.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index b5f4c127a1..526d2fe674 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -31,7 +31,9 @@ def __init__(self, filter_data: dict, defaults: Defaults | None = None): self.updated_at = arrow.get(filter_data["updated_at"]) self.actions, self.validations = create_settings(filter_data["settings"], defaults=defaults) if self.extra_fields_type: - self.extra_fields = self.extra_fields_type.parse_raw(filter_data["additional_field"] or "{}") # noqa: P103 + self.extra_fields = self.extra_fields_type.parse_raw( + filter_data["additional_settings"] or "{}" # noqa: P103 + ) else: self.extra_fields = None diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 2a7f8f81f0..efea57a6a8 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1177,7 +1177,7 @@ async def _post_new_filter( description = description or None payload = { "filter_list": list_id, "content": content, "description": description, - "additional_field": json.dumps(filter_settings), **settings + "additional_settings": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.post('bot/filter/filters', json=to_serializable(payload)) new_filter = filter_list.add_filter(list_type, response) @@ -1220,7 +1220,7 @@ async def _patch_filter( description = description or None payload = { "filter_list": list_id, "content": content, "description": description, - "additional_field": json.dumps(filter_settings), **settings + "additional_settings": json.dumps(filter_settings), **settings } response = await bot.instance.api_client.patch( f'bot/filter/filters/{filter_.id}', json=to_serializable(payload) diff --git a/tests/bot/exts/filtering/test_discord_token_filter.py b/tests/bot/exts/filtering/test_discord_token_filter.py index ef124e6ffb..4d7e69bdc4 100644 --- a/tests/bot/exts/filtering/test_discord_token_filter.py +++ b/tests/bot/exts/filtering/test_discord_token_filter.py @@ -22,7 +22,7 @@ def setUp(self): "content": "discord_token", "description": None, "settings": {}, - "additional_field": "{}", # noqa: P103 + "additional_settings": "{}", # noqa: P103 "created_at": now, "updated_at": now }) diff --git a/tests/bot/exts/filtering/test_extension_filter.py b/tests/bot/exts/filtering/test_extension_filter.py index 351daa0b4c..52506d0be5 100644 --- a/tests/bot/exts/filtering/test_extension_filter.py +++ b/tests/bot/exts/filtering/test_extension_filter.py @@ -25,7 +25,7 @@ def setUp(self): for i, filter_content in enumerate(self.whitelist, start=1): filters.append({ "id": i, "content": filter_content, "description": None, "settings": {}, - "additional_field": "{}", "created_at": now, "updated_at": now # noqa: P103 + "additional_settings": "{}", "created_at": now, "updated_at": now # noqa: P103 }) self.filter_list.add_list({ "id": 1, diff --git a/tests/bot/exts/filtering/test_token_filter.py b/tests/bot/exts/filtering/test_token_filter.py index 0dfc8ae9fe..82cc6b67e8 100644 --- a/tests/bot/exts/filtering/test_token_filter.py +++ b/tests/bot/exts/filtering/test_token_filter.py @@ -40,7 +40,7 @@ async def test_token_filter_triggers(self): "content": pattern, "description": None, "settings": {}, - "additional_field": "{}", # noqa: P103 + "additional_settings": "{}", # noqa: P103 "created_at": now, "updated_at": now }) From 16ddb323f9b56d6ca91deebeec5aa8c85c899450 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 26 Mar 2023 23:34:32 +0300 Subject: [PATCH 112/132] List input fixes --- bot/exts/filtering/_ui/ui.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 8cd2864a93..859a1eff79 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -385,7 +385,9 @@ async def apply_addition(self, interaction: Interaction, item: str) -> None: async def apply_edit(self, interaction: Interaction, new_list: str) -> None: """Change the contents of the list.""" self.stored_value = list(set(part.strip() for part in new_list.split(",") if part.strip())) - await interaction.response.edit_message(content=f"Current list: {self.stored_value}", view=self.copy()) + await interaction.response.edit_message( + content=f"Current list: [{', '.join(self.stored_value)}]", view=self.copy() + ) self.stop() @discord.ui.button(label="Add Value") @@ -468,7 +470,7 @@ async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.S await interaction.response.send_message(f"Choose a value for `{setting_name}`:", view=view, ephemeral=True) elif type_ in (set, list, tuple): if (current_value := self.current_value(setting_name)) is not MISSING: - current_list = list(current_value) + current_list = [str(elem) for elem in current_value] else: current_list = [] await interaction.response.send_message( From c87b7560e6a208614440b1607a08d5b6479cf4a2 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 26 Mar 2023 23:56:57 +0300 Subject: [PATCH 113/132] Address typehint issues and IDE complaints --- bot/exts/filtering/_filter_lists/filter_list.py | 4 ++-- bot/exts/filtering/_filters/filter.py | 4 ++-- bot/exts/filtering/_settings.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index bf02071cfb..d4c975766a 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -1,6 +1,6 @@ import dataclasses import typing -from abc import abstractmethod +from abc import ABC, abstractmethod from collections import defaultdict from collections.abc import Iterable from dataclasses import dataclass @@ -258,7 +258,7 @@ async def filter_list_result(self, ctx: FilterContext) -> list[Filter]: return await self._create_filter_list_result(ctx, self.defaults, event_filters) -class UniquesListBase(FilterList[UniqueFilter]): +class UniquesListBase(FilterList[UniqueFilter], ABC): """ A list of unique filters. diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 526d2fe674..128e84645e 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -1,4 +1,4 @@ -from abc import abstractmethod +from abc import ABC, abstractmethod from typing import Any import arrow @@ -86,7 +86,7 @@ def __str__(self) -> str: return string -class UniqueFilter(Filter): +class UniqueFilter(Filter, ABC): """ Unique filters are ones that should only be run once in a given context. diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index 75e810df59..f472c5d1df 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -69,7 +69,7 @@ class Settings(FieldRequiring, dict[str, T]): the filter list which contains the filter. """ - entry_type = T + entry_type: type[T] _already_warned: set[str] = set() From 0df028aebbb8fcfd8f6793bd84ddcca0ebf1f16f Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 27 Mar 2023 18:52:37 +0300 Subject: [PATCH 114/132] Fix condition --- bot/exts/filtering/_filter_lists/antispam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index ba20051fc9..e76f953388 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -162,7 +162,7 @@ async def send_alert(self, antispam_list: AntispamList) -> None: add, (other_ctx.action_descriptions for other_ctx in other_contexts), ctx.action_descriptions ) # It shouldn't ever come to this, but just in case. - if descriptions_num := len(new_ctx.action_descriptions) > 20: + if (descriptions_num := len(new_ctx.action_descriptions)) > 20: new_ctx.action_descriptions = new_ctx.action_descriptions[:20] new_ctx.action_descriptions[-1] += f" (+{descriptions_num - 20} other actions)" new_ctx.related_messages = reduce( From 7a1a553b67ff7fc300e9b457f8750efbf28ecf96 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 27 Mar 2023 19:01:27 +0300 Subject: [PATCH 115/132] Group identical action descriptions in antispam --- bot/exts/filtering/_filter_lists/antispam.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index e76f953388..9745372af8 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -1,5 +1,6 @@ import asyncio import typing +from collections import Counter from collections.abc import Callable, Coroutine from dataclasses import dataclass, field from datetime import timedelta @@ -158,9 +159,12 @@ async def send_alert(self, antispam_list: AntispamList) -> None: ctx, *other_contexts = self.contexts new_ctx = FilterContext(ctx.event, ctx.author, ctx.channel, ctx.content, ctx.message) - new_ctx.action_descriptions = reduce( + all_descriptions_counts = Counter(reduce( add, (other_ctx.action_descriptions for other_ctx in other_contexts), ctx.action_descriptions - ) + )) + new_ctx.action_descriptions = [ + f"{action} X {count}" if count > 1 else action for action, count in all_descriptions_counts.items() + ] # It shouldn't ever come to this, but just in case. if (descriptions_num := len(new_ctx.action_descriptions)) > 20: new_ctx.action_descriptions = new_ctx.action_descriptions[:20] From a62993b44cfc0f2c3c5d746005b9a716dea82add Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 27 Mar 2023 19:03:13 +0300 Subject: [PATCH 116/132] Correct filter match docstring --- bot/exts/filtering/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index efea57a6a8..58d2f125ef 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -599,7 +599,7 @@ async def f_match( self, ctx: Context, no_user: bool | None, message: Message | None, *, string: str | None ) -> None: """ - Post any responses from the filter lists for the given message or string. + List the filters triggered for the given message or string. If there's a `message`, the `string` will be ignored. Note that if a `message` is provided, it will go through all validations appropriate to where it was sent and who sent it. To check for matches regardless of the author From f01883682f4d333382d8e8a89363dc906fe86342 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 28 Mar 2023 01:12:25 +0300 Subject: [PATCH 117/132] Support custom value representation in filtering UI Adds the `CustomIOField` class which can be used as a base for wrappers that store a value with a customized way to process the user input and to present the value in the UI. --- .../actions/infraction_and_notification.py | 52 +++++++++++++-- bot/exts/filtering/_ui/filter.py | 10 +-- bot/exts/filtering/_ui/search.py | 4 +- bot/exts/filtering/_utils.py | 66 +++++++++++++++++-- bot/exts/filtering/filtering.py | 8 +-- .../exts/filtering/test_settings_entries.py | 16 +++-- 6 files changed, 127 insertions(+), 29 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 5ae4901b62..e3df470298 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -1,9 +1,9 @@ -from datetime import timedelta from enum import Enum, auto from typing import ClassVar import arrow import discord.abc +from dateutil.relativedelta import relativedelta from discord import Colour, Embed, Member, User from discord.errors import Forbidden from pydantic import validator @@ -15,7 +15,8 @@ from bot.constants import Channels from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry -from bot.exts.filtering._utils import FakeContext +from bot.exts.filtering._utils import CustomIOField, FakeContext +from bot.utils.time import humanize_delta, parse_duration_string, relativedelta_to_timedelta log = get_logger(__name__) @@ -31,6 +32,38 @@ } +class InfractionDuration(CustomIOField): + """A field that converts a string to a duration and presents it in a human-readable format.""" + + @classmethod + def process_value(cls, v: str | relativedelta) -> relativedelta: + """ + Transform the given string into a relativedelta. + + Raise a ValueError if the conversion is not possible. + """ + if isinstance(v, relativedelta): + return v + + try: + v = float(v) + except ValueError: # Not a float. + if not (delta := parse_duration_string(v)): + raise ValueError(f"`{v}` is not a valid duration string.") + else: + delta = relativedelta(seconds=float(v)).normalized() + + return delta + + def serialize(self) -> float: + """The serialized value is the total number of seconds this duration represents.""" + return relativedelta_to_timedelta(self.value).total_seconds() + + def __str__(self): + """Represent the stored duration in a human-readable format.""" + return humanize_delta(self.value, max_units=2) if self.value else "Permanent" + + class Infraction(Enum): """An enumeration of infraction types. The lower the value, the higher it is on the hierarchy.""" @@ -53,7 +86,7 @@ async def invoke( message: discord.Message, channel: discord.abc.GuildChannel | discord.DMChannel, alerts_channel: discord.TextChannel, - duration: float, + duration: InfractionDuration, reason: str ) -> None: """Invokes the command matching the infraction name.""" @@ -72,7 +105,7 @@ async def invoke( if self.name in ("KICK", "WARNING", "WATCH", "NOTE"): await command(ctx, user, reason=reason or None) else: - duration = arrow.utcnow() + timedelta(seconds=duration) if duration else None + duration = arrow.utcnow().datetime + duration.value if duration.value else None await command(ctx, user, duration, reason=reason or None) @@ -91,7 +124,10 @@ class InfractionAndNotification(ActionEntry): "the harsher one will be applied (by type or duration).\n\n" "Valid infraction types in order of harshness: " ) + ", ".join(infraction.name for infraction in Infraction), - "infraction_duration": "How long the infraction should last for in seconds. 0 for permanent.", + "infraction_duration": ( + "How long the infraction should last for in seconds. 0 for permanent. " + "Also supports durations as in an infraction invocation (such as `10d`)." + ), "infraction_reason": "The reason delivered with the infraction.", "infraction_channel": ( "The channel ID in which to invoke the infraction (and send the confirmation message). " @@ -106,7 +142,7 @@ class InfractionAndNotification(ActionEntry): dm_embed: str infraction_type: Infraction infraction_reason: str - infraction_duration: float + infraction_duration: InfractionDuration infraction_channel: int @validator("infraction_type", pre=True) @@ -184,8 +220,10 @@ def union(self, other: Self) -> Self: result = other.copy() other = self else: + now = arrow.utcnow().datetime if self.infraction_duration is None or ( - other.infraction_duration is not None and self.infraction_duration > other.infraction_duration + other.infraction_duration is not None + and now + self.infraction_duration.value > now + other.infraction_duration.value ): result = self.copy() else: diff --git a/bot/exts/filtering/_ui/filter.py b/bot/exts/filtering/_ui/filter.py index 1ef25f17a7..5b23b71e9c 100644 --- a/bot/exts/filtering/_ui/filter.py +++ b/bot/exts/filtering/_ui/filter.py @@ -33,7 +33,7 @@ def build_filter_repr_dict( default_setting_values = {} for settings_group in filter_list[list_type].defaults: for _, setting in settings_group.items(): - default_setting_values.update(to_serializable(setting.dict())) + default_setting_values.update(to_serializable(setting.dict(), ui_repr=True)) # Add overrides. It's done in this way to preserve field order, since the filter won't have all settings. total_values = {} @@ -434,10 +434,10 @@ def description_and_settings_converter( return description, settings, filter_settings -def filter_serializable_overrides(filter_: Filter) -> tuple[dict, dict]: - """Get a serializable version of the filter's overrides.""" +def filter_overrides_for_ui(filter_: Filter) -> tuple[dict, dict]: + """Get the filter's overrides in a format that can be displayed in the UI.""" overrides_values, extra_fields_overrides = filter_.overrides - return to_serializable(overrides_values), to_serializable(extra_fields_overrides) + return to_serializable(overrides_values, ui_repr=True), to_serializable(extra_fields_overrides, ui_repr=True) def template_settings( @@ -461,4 +461,4 @@ def template_settings( raise BadArgument( f"The template filter name is {filter_.name!r}, but the target filter is {filter_type.name!r}" ) - return filter_serializable_overrides(filter_) + return filter_.overrides diff --git a/bot/exts/filtering/_ui/search.py b/bot/exts/filtering/_ui/search.py index d553c28eaa..dba7f3cea6 100644 --- a/bot/exts/filtering/_ui/search.py +++ b/bot/exts/filtering/_ui/search.py @@ -10,7 +10,7 @@ from bot.exts.filtering._filter_lists import FilterList, ListType from bot.exts.filtering._filters.filter import Filter from bot.exts.filtering._settings_types.settings_entry import SettingsEntry -from bot.exts.filtering._ui.filter import filter_serializable_overrides +from bot.exts.filtering._ui.filter import filter_overrides_for_ui from bot.exts.filtering._ui.ui import ( COMPONENT_TIMEOUT, CustomCallbackSelect, EditBaseView, MISSING, SETTINGS_DELIMITER, parse_value, populate_embed_from_dict @@ -114,7 +114,7 @@ def template_settings( if filter_type and not isinstance(filter_, filter_type): raise BadArgument(f"The filter with ID `{filter_id}` is not of type {filter_type.name!r}.") - settings, filter_settings = filter_serializable_overrides(filter_) + settings, filter_settings = filter_overrides_for_ui(filter_) return settings, filter_settings, type(filter_) diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index da433330f5..a43233f20e 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import importlib import importlib.util import inspect @@ -12,6 +14,7 @@ import discord import regex from discord.ext.commands import Command +from typing_extensions import Self import bot from bot.bot import Bot @@ -24,6 +27,8 @@ T = TypeVar('T') +Serializable = Union[bool, int, float, str, list, dict, None] + def subclasses_in_package(package: str, prefix: str, parent: T) -> set[T]: """Return all the subclasses of class `parent`, found in the top-level of `package`, given by absolute path.""" @@ -62,8 +67,13 @@ def past_tense(word: str) -> str: return word + "ed" -def to_serializable(item: Any) -> Union[bool, int, float, str, list, dict, None]: - """Convert the item into an object that can be converted to JSON.""" +def to_serializable(item: Any, *, ui_repr: bool = False) -> Serializable: + """ + Convert the item into an object that can be converted to JSON. + + `ui_repr` dictates whether to use the UI representation of `CustomIOField` instances (if any) + or the DB-oriented representation. + """ if isinstance(item, (bool, int, float, str, type(None))): return item if isinstance(item, dict): @@ -71,10 +81,12 @@ def to_serializable(item: Any) -> Union[bool, int, float, str, list, dict, None] for key, value in item.items(): if not isinstance(key, (bool, int, float, str, type(None))): key = str(key) - result[key] = to_serializable(value) + result[key] = to_serializable(value, ui_repr=ui_repr) return result if isinstance(item, Iterable): - return [to_serializable(subitem) for subitem in item] + return [to_serializable(subitem, ui_repr=ui_repr) for subitem in item] + if not ui_repr and hasattr(item, "serialize"): + return item.serialize() return str(item) @@ -222,3 +234,49 @@ def __post_init__(self): async def send(self, *args, **kwargs) -> discord.Message: """A wrapper for channel.send.""" return await self.channel.send(*args, **kwargs) + + +class CustomIOField: + """ + A class to be used as a data type in SettingEntry subclasses. + + Its subclasses can have custom methods to read and represent the value, which will be used by the UI. + """ + + def __init__(self, value: Any): + self.value = self.process_value(value) + + @classmethod + def __get_validators__(cls): + """Boilerplate for Pydantic.""" + yield cls.validate + + @classmethod + def validate(cls, v: Any) -> Self: + """Takes the given value and returns a class instance with that value.""" + if isinstance(v, CustomIOField): + return cls(v.value) + + return cls(v) + + def __eq__(self, other: CustomIOField): + if not isinstance(other, CustomIOField): + return NotImplemented + return self.value == other.value + + @classmethod + def process_value(cls, v: str) -> Any: + """ + Perform any necessary transformations before the value is stored in a new instance. + + Override this method to customize the input behavior. + """ + return v + + def serialize(self) -> Serializable: + """Override this method to customize how the value will be serialized.""" + return self.value + + def __str__(self): + """Override this method to change how the value will be displayed by the UI.""" + return self.value diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 58d2f125ef..8fd4ddb132 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -31,7 +31,7 @@ from bot.exts.filtering._settings import ActionSettings from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction from bot.exts.filtering._ui.filter import ( - build_filter_repr_dict, description_and_settings_converter, filter_serializable_overrides, populate_embed_from_dict + build_filter_repr_dict, description_and_settings_converter, filter_overrides_for_ui, populate_embed_from_dict ) from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter from bot.exts.filtering._ui.search import SearchEditView, search_criteria_converter @@ -383,7 +383,7 @@ async def filter(self, ctx: Context, id_: Optional[int] = None) -> None: return filter_, filter_list, list_type = result - overrides_values, extra_fields_overrides = filter_serializable_overrides(filter_) + overrides_values, extra_fields_overrides = filter_overrides_for_ui(filter_) all_settings_repr_dict = build_filter_repr_dict( filter_list, list_type, type(filter_), overrides_values, extra_fields_overrides @@ -493,7 +493,7 @@ async def f_edit( return filter_, filter_list, list_type = result filter_type = type(filter_) - settings, filter_settings = filter_serializable_overrides(filter_) + settings, filter_settings = filter_overrides_for_ui(filter_) description, new_settings, new_filter_settings = description_and_settings_converter( filter_list, list_type, filter_type, @@ -734,7 +734,7 @@ async def fl_describe( setting_values = {} for settings_group in filter_list[list_type].defaults: for _, setting in settings_group.items(): - setting_values.update(to_serializable(setting.dict())) + setting_values.update(to_serializable(setting.dict(), ui_repr=True)) embed = Embed(colour=Colour.blue()) populate_embed_from_dict(embed, setting_values) diff --git a/tests/bot/exts/filtering/test_settings_entries.py b/tests/bot/exts/filtering/test_settings_entries.py index c5f0152b0f..3ae0b5ab56 100644 --- a/tests/bot/exts/filtering/test_settings_entries.py +++ b/tests/bot/exts/filtering/test_settings_entries.py @@ -1,7 +1,9 @@ import unittest from bot.exts.filtering._filter_context import Event, FilterContext -from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction, InfractionAndNotification +from bot.exts.filtering._settings_types.actions.infraction_and_notification import ( + Infraction, InfractionAndNotification, InfractionDuration +) from bot.exts.filtering._settings_types.validations.bypass_roles import RoleBypass from bot.exts.filtering._settings_types.validations.channel_scope import ChannelScope from bot.exts.filtering._settings_types.validations.filter_dm import FilterDM @@ -154,7 +156,7 @@ def test_infraction_merge_of_same_infraction_type(self): infraction1 = InfractionAndNotification( infraction_type="TIMEOUT", infraction_reason="hi", - infraction_duration=10, + infraction_duration=InfractionDuration(10), dm_content="how", dm_embed="what is", infraction_channel=0 @@ -162,7 +164,7 @@ def test_infraction_merge_of_same_infraction_type(self): infraction2 = InfractionAndNotification( infraction_type="TIMEOUT", infraction_reason="there", - infraction_duration=20, + infraction_duration=InfractionDuration(20), dm_content="are you", dm_embed="your name", infraction_channel=0 @@ -175,7 +177,7 @@ def test_infraction_merge_of_same_infraction_type(self): { "infraction_type": Infraction.TIMEOUT, "infraction_reason": "there", - "infraction_duration": 20.0, + "infraction_duration": InfractionDuration(20.0), "dm_content": "are you", "dm_embed": "your name", "infraction_channel": 0 @@ -187,7 +189,7 @@ def test_infraction_merge_of_different_infraction_types(self): infraction1 = InfractionAndNotification( infraction_type="TIMEOUT", infraction_reason="hi", - infraction_duration=20, + infraction_duration=InfractionDuration(20), dm_content="", dm_embed="", infraction_channel=0 @@ -195,7 +197,7 @@ def test_infraction_merge_of_different_infraction_types(self): infraction2 = InfractionAndNotification( infraction_type="BAN", infraction_reason="", - infraction_duration=10, + infraction_duration=InfractionDuration(10), dm_content="there", dm_embed="", infraction_channel=0 @@ -208,7 +210,7 @@ def test_infraction_merge_of_different_infraction_types(self): { "infraction_type": Infraction.BAN, "infraction_reason": "", - "infraction_duration": 10.0, + "infraction_duration": InfractionDuration(10), "dm_content": "there", "dm_embed": "", "infraction_channel": 0 From 990d716ea2e3532e305460d1aef88f9cba211614 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 01:32:43 +0300 Subject: [PATCH 118/132] Code style and clarifications --- bot/exts/filtering/FILTERS-DEVELOPMENT.md | 2 +- bot/exts/filtering/_filter_lists/antispam.py | 16 ++++++++-------- bot/exts/filtering/_filter_lists/extension.py | 10 +++------- bot/exts/filtering/_settings.py | 2 +- .../actions/infraction_and_notification.py | 1 + .../_settings_types/validations/channel_scope.py | 4 +++- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/bot/exts/filtering/FILTERS-DEVELOPMENT.md b/bot/exts/filtering/FILTERS-DEVELOPMENT.md index d5896d5565..c6237b60c0 100644 --- a/bot/exts/filtering/FILTERS-DEVELOPMENT.md +++ b/bot/exts/filtering/FILTERS-DEVELOPMENT.md @@ -9,7 +9,7 @@ There are several types of filters, and two filters of the same type differ by t For example, filters of type "token" search for a specific token inside the provided string. One token filter might look for the string "lemon", while another will look for the string "joe". -Each filter has a set of settings that decide when it triggers (e.g. in which channels), and what happens if it does (e.g. delete the message). +Each filter has a set of settings that decide when it triggers (e.g. in which channels, in which categories, etc.), and what happens if it does (e.g. delete the message, ping specific roles/users, etc.). Filters of a specific type can have additional settings that are special to them. A list of filters is contained within a filter list. diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py index 9745372af8..94f80e6ebe 100644 --- a/bot/exts/filtering/_filter_lists/antispam.py +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -18,7 +18,7 @@ from bot.exts.filtering._filters.antispam import antispam_filter_types from bot.exts.filtering._filters.filter import Filter, UniqueFilter from bot.exts.filtering._settings import ActionSettings -from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction +from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction, InfractionAndNotification from bot.exts.filtering._ui.ui import AlertView, build_mod_alert if typing.TYPE_CHECKING: @@ -93,22 +93,22 @@ async def actions_for( current_actions.pop("ping", None) current_actions.pop("send_alert", None) - new_infraction = current_actions["infraction_and_notification"].copy() + new_infraction = current_actions[InfractionAndNotification.name].copy() # Smaller infraction value => higher in hierarchy. if not current_infraction or new_infraction.infraction_type.value < current_infraction.value: # Pick the first triggered filter for the reason, there's no good way to decide between them. new_infraction.infraction_reason = ( f"{triggers[0].name.replace('_', ' ')} spam – {ctx.filter_info[triggers[0]]}" ) - current_actions["infraction_and_notification"] = new_infraction + current_actions[InfractionAndNotification.name] = new_infraction self.message_deletion_queue[ctx.author].current_infraction = new_infraction.infraction_type else: - current_actions.pop("infraction_and_notification", None) + current_actions.pop(InfractionAndNotification.name, None) # Provide some message in case another filter list wants there to be an alert. return current_actions, ["Handling spam event..."], {ListType.DENY: triggers} - def _create_deletion_context_handler(self, context_id: Member) -> Callable[[FilterContext], Coroutine]: + def _create_deletion_context_handler(self, member: Member) -> Callable[[FilterContext], Coroutine]: async def schedule_processing(ctx: FilterContext) -> None: """ Schedule a coroutine to process the deletion context. @@ -123,11 +123,11 @@ async def process_deletion_context() -> None: log.trace("Sleeping before processing message deletion queue.") await asyncio.sleep(ALERT_DELAY) - if context_id not in self.message_deletion_queue: - log.error(f"Started processing deletion queue for context `{context_id}`, but it was not found!") + if member not in self.message_deletion_queue: + log.error(f"Started processing deletion queue for context `{member}`, but it was not found!") return - deletion_context = self.message_deletion_queue.pop(context_id) + deletion_context = self.message_deletion_queue.pop(member) await deletion_context.send_alert(self) scheduling.create_task(process_deletion_context()) diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index dda2c3179f..9d6c4c0e7c 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -27,7 +27,7 @@ ) DISALLOWED_EMBED_DESCRIPTION = ( - "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " + "It looks like you tried to attach file type(s) that we do not allow ({joined_blacklist}). " "We currently allow the following file types: **{joined_whitelist}**.\n\n" "Feel free to ask in {meta_channel_mention} if you think this is a mistake." ) @@ -99,11 +99,7 @@ async def actions_for( ctx.dm_embed = PY_EMBED_DESCRIPTION elif txt_extensions := {ext for ext in TXT_LIKE_FILES if ext in not_allowed}: # Work around Discord auto-conversion of messages longer than 2000 chars to .txt - cmd_channel = bot.instance.get_channel(Channels.bot_commands) - ctx.dm_embed = TXT_EMBED_DESCRIPTION.format( - blocked_extension=txt_extensions.pop(), - cmd_channel_mention=cmd_channel.mention - ) + ctx.dm_embed = TXT_EMBED_DESCRIPTION.format(blocked_extension=txt_extensions.pop()) else: meta_channel = bot.instance.get_channel(Channels.meta) if not self._whitelisted_description: @@ -112,7 +108,7 @@ async def actions_for( ) ctx.dm_embed = DISALLOWED_EMBED_DESCRIPTION.format( joined_whitelist=self._whitelisted_description, - blocked_extensions_str=", ".join(not_allowed), + joined_blacklist=", ".join(not_allowed), meta_channel_mention=meta_channel.mention, ) diff --git a/bot/exts/filtering/_settings.py b/bot/exts/filtering/_settings.py index f472c5d1df..f51a427049 100644 --- a/bot/exts/filtering/_settings.py +++ b/bot/exts/filtering/_settings.py @@ -73,7 +73,7 @@ class Settings(FieldRequiring, dict[str, T]): _already_warned: set[str] = set() - @abstractmethod + @abstractmethod # ABCs have to have at least once abstract method to actually count as such. def __init__(self, settings_data: dict, *, defaults: Settings | None = None, keep_empty: bool = False): super().__init__() diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index e3df470298..3b59c2feb3 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -232,6 +232,7 @@ def union(self, other: Self) -> Self: # If the winner has no message but the loser does, copy the message to the winner. result_overrides = result.overrides + # Either take both or nothing, don't mix content from one filter and embed from another. if "dm_content" not in result_overrides and "dm_embed" not in result_overrides: other_overrides = other.overrides if "dm_content" in other_overrides: diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index d37efaa099..737465137b 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -10,7 +10,7 @@ class ChannelScope(ValidationEntry): """A setting entry which tells whether the filter was invoked in a whitelisted channel or category.""" name: ClassVar[str] = "channel_scope" - description: ClassVar[str] = { + description: ClassVar[dict[str, str]] = { "disabled_channels": ( "A list of channel IDs or channel names. " "The filter will not trigger in these channels even if the category is expressly enabled." @@ -29,6 +29,8 @@ class ChannelScope(ValidationEntry): ) } + # NOTE: Don't change this to use the new 3.10 union syntax unless you ensure Pydantic type validation and coercion + # work properly. At the time of writing this code there's a difference. disabled_channels: set[Union[int, str]] disabled_categories: set[Union[int, str]] enabled_channels: set[Union[int, str]] From 9911125b7dbd173669760ddbd0222b42739bba13 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 01:43:37 +0300 Subject: [PATCH 119/132] Properly format alert when there's no file extension --- bot/exts/filtering/_filter_lists/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_filter_lists/extension.py b/bot/exts/filtering/_filter_lists/extension.py index 9d6c4c0e7c..d805fa7aa2 100644 --- a/bot/exts/filtering/_filter_lists/extension.py +++ b/bot/exts/filtering/_filter_lists/extension.py @@ -115,4 +115,4 @@ async def actions_for( ctx.matches += not_allowed.values() ctx.blocked_exts |= set(not_allowed) actions = self[ListType.ALLOW].defaults.actions if ctx.event != Event.SNEKBOX else None - return actions, [f"`{ext}`" for ext in not_allowed], {ListType.ALLOW: triggered} + return actions, [f"`{ext}`" if ext else "`No Extension`" for ext in not_allowed], {ListType.ALLOW: triggered} From f693bb27452dc6b83e5eab804ed95ca1d3f9e63b Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 01:49:59 +0300 Subject: [PATCH 120/132] Handle server possibly not having an icon --- bot/exts/filtering/_filter_lists/invite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/exts/filtering/_filter_lists/invite.py b/bot/exts/filtering/_filter_lists/invite.py index b9732a6dc9..d934e9d535 100644 --- a/bot/exts/filtering/_filter_lists/invite.py +++ b/bot/exts/filtering/_filter_lists/invite.py @@ -140,8 +140,9 @@ def _guild_embed(invite: Invite) -> Embed: embed = Embed() if invite.guild: embed.title = invite.guild.name - embed.set_thumbnail(url=invite.guild.icon.url) embed.set_footer(text=f"Guild ID: {invite.guild.id}") + if invite.guild.icon is not None: + embed.set_thumbnail(url=invite.guild.icon.url) else: embed.title = "Group DM" From 0a51bbe41279ac884b0e713c53590e612b79683a Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 01:57:35 +0300 Subject: [PATCH 121/132] Handle infracted user not being on the server This shouldn't happen in production, but just in case --- .../_settings_types/actions/infraction_and_notification.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 3b59c2feb3..a7fb37ca90 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -101,6 +101,13 @@ async def invoke( member = await get_or_fetch_member(channel.guild, user.id) if member: user = member + else: + log.warning( + f"The user {user} were set to receive an automatic {command_name}, " + "but they were not found in the guild." + ) + return + ctx = FakeContext(message, channel, command) if self.name in ("KICK", "WARNING", "WATCH", "NOTE"): await command(ctx, user, reason=reason or None) From 7415131908c76d4398ac73294fdddb522e10fa9b Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 02:12:51 +0300 Subject: [PATCH 122/132] `domain/subdomains` -> `domain/only_subdomains` for clarity Also changes the alert to use the full domain instead of the filter content. --- bot/exts/filtering/_filters/domain.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/exts/filtering/_filters/domain.py b/bot/exts/filtering/_filters/domain.py index 7c229fdcb3..ac9cc90189 100644 --- a/bot/exts/filtering/_filters/domain.py +++ b/bot/exts/filtering/_filters/domain.py @@ -1,5 +1,5 @@ import re -from typing import ClassVar, Optional +from typing import ClassVar from urllib.parse import urlparse import tldextract @@ -15,12 +15,12 @@ class ExtraDomainSettings(BaseModel): """Extra settings for how domains should be matched in a message.""" - subdomains_description: ClassVar[str] = ( - "A boolean. If True, will will only trigger for subdomains and subpaths, and not for the domain itself." + only_subdomains_description: ClassVar[str] = ( + "A boolean. If True, will only trigger for subdomains and subpaths, and not for the domain itself." ) # Whether to trigger only for subdomains and subpaths, and not the specified domain itself. - subdomains: Optional[bool] = False + only_subdomains: bool = False class DomainFilter(Filter): @@ -41,10 +41,10 @@ async def triggered_on(self, ctx: FilterContext) -> bool: for found_url in ctx.content: extract = tldextract.extract(found_url) if self.content in found_url and extract.registered_domain == domain: - if self.extra_fields.subdomains: + if self.extra_fields.only_subdomains: if not extract.subdomain and not urlparse(f"https://{found_url}").path: return False - ctx.matches.append(self.content) + ctx.matches.append(found_url) ctx.notification_domain = self.content return True return False From 43e2a77a3ad82579753cc0fd7415f24a4dbbdea3 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 02:15:58 +0300 Subject: [PATCH 123/132] Escape markdown in alert matches --- bot/exts/filtering/_ui/ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 859a1eff79..7ef21a5ce2 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -102,7 +102,7 @@ async def build_mod_alert(ctx: FilterContext, triggered_filters: dict[FilterList filters.append(f"**{filter_list.name.title()} Filters:** {', '.join(list_message)}") filters = "\n".join(filters) - matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) if ctx.matches else "" + matches = "**Matches:** " + escape_markdown(", ".join(repr(match) for match in ctx.matches)) if ctx.matches else "" actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") mod_alert_message = "\n".join(part for part in (triggered_by, triggered_in, filters, matches, actions) if part) From 6d5d96434687c66c637328ef546586441c57c8fc Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 02:29:32 +0300 Subject: [PATCH 124/132] Specify edited setting name in confirmation message --- bot/exts/filtering/_ui/ui.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 7ef21a5ce2..6a9328474f 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -48,6 +48,8 @@ SETTINGS_DELIMITER = re.compile(r"\s+(?=\S+=\S+)") SINGLE_SETTING_PATTERN = re.compile(r"[\w/]+=.+") +EDIT_CONFIRMED_MESSAGE = "✅ Edit for `{0}` confirmed" + # Sentinel value to denote that a value is missing MISSING = object() @@ -273,9 +275,9 @@ def __init__(self, setting_name: str, update_callback: Callable): async def callback(self, interaction: Interaction) -> Any: """Respond to the interaction by sending the boolean value to the update callback.""" - await interaction.response.edit_message(content="✅ Edit confirmed", view=None) value = self.values[0] == "True" await self.update_callback(setting_name=self.setting_name, setting_value=value) + await interaction.response.edit_message(content=EDIT_CONFIRMED_MESSAGE.format(self.setting_name), view=None) def __init__(self, setting_name: str, update_callback: Callable): super().__init__(timeout=COMPONENT_TIMEOUT) @@ -309,8 +311,10 @@ async def on_submit(self, interaction: Interaction) -> None: f"Could not process the input value for `{self.setting_name}`.", ephemeral=True ) else: - await interaction.response.defer() await self.update_callback(setting_name=self.setting_name, setting_value=value) + await interaction.response.send_message( + content=EDIT_CONFIRMED_MESSAGE.format(self.setting_name), ephemeral=True + ) class SequenceEditView(discord.ui.View): @@ -404,8 +408,8 @@ async def free_input(self, interaction: Interaction, button: discord.ui.Button) async def confirm(self, interaction: Interaction, button: discord.ui.Button) -> None: """Send the final value to the embed editor.""" # Edit first, it might time out otherwise. - await interaction.response.edit_message(content="✅ Edit confirmed", view=None) await self.update_callback(setting_name=self.setting_name, setting_value=self.stored_value) + await interaction.response.edit_message(content=EDIT_CONFIRMED_MESSAGE.format(self.setting_name), view=None) self.stop() @discord.ui.button(label="🚫 Cancel", style=discord.ButtonStyle.red) @@ -433,8 +437,8 @@ def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Calla async def callback(self, interaction: Interaction) -> Any: """Respond to the interaction by sending the enum value to the update callback.""" - await interaction.response.edit_message(content="✅ Edit confirmed", view=None) await self.update_callback(setting_name=self.setting_name, setting_value=self.values[0]) + await interaction.response.edit_message(content=EDIT_CONFIRMED_MESSAGE.format(self.setting_name), view=None) def __init__(self, setting_name: str, enum_cls: EnumMeta, update_callback: Callable): super().__init__(timeout=COMPONENT_TIMEOUT) From 93b3d19b535e56c32d7a0e15803c7fc1038dc825 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 03:11:16 +0300 Subject: [PATCH 125/132] Fix infraction duration display for filter lists --- bot/exts/filtering/_ui/filter_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_ui/filter_list.py b/bot/exts/filtering/_ui/filter_list.py index a4526f090f..4d6f76a89c 100644 --- a/bot/exts/filtering/_ui/filter_list.py +++ b/bot/exts/filtering/_ui/filter_list.py @@ -49,7 +49,7 @@ def build_filterlist_repr_dict(filter_list: FilterList, list_type: ListType, new default_setting_values = {} for settings_group in filter_list[list_type].defaults: for _, setting in settings_group.items(): - default_setting_values.update(to_serializable(setting.dict())) + default_setting_values.update(to_serializable(setting.dict(), ui_repr=True)) # Add new values. It's done in this way to preserve field order, since the new_values won't have all settings. total_values = {} From 7386e6b1d6ebaea727971fd25ff12840b0c7a435 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Wed, 5 Apr 2023 03:15:17 +0300 Subject: [PATCH 126/132] Fix test --- tests/bot/exts/filtering/test_extension_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/exts/filtering/test_extension_filter.py b/tests/bot/exts/filtering/test_extension_filter.py index 52506d0be5..800fad3a07 100644 --- a/tests/bot/exts/filtering/test_extension_filter.py +++ b/tests/bot/exts/filtering/test_extension_filter.py @@ -115,7 +115,7 @@ async def test_other_disallowed_extension_embed_description(self): ctx.dm_embed, extension.DISALLOWED_EMBED_DESCRIPTION.format( joined_whitelist=", ".join(self.whitelist), - blocked_extensions_str=".disallowed", + joined_blacklist=".disallowed", meta_channel_mention=meta_channel.mention ) ) From 076c2e910cdac0f168084caaf7d1331fc40a638a Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Thu, 6 Apr 2023 01:10:33 +0300 Subject: [PATCH 127/132] Adjust to site using dicts in additional_settings instead of JSON strings This is to make sure that the unique constraint always compares between dicts instead of sometimes dealing with nulls. --- bot/exts/filtering/_filters/filter.py | 4 +--- bot/exts/filtering/filtering.py | 4 ++-- tests/bot/exts/filtering/test_discord_token_filter.py | 2 +- tests/bot/exts/filtering/test_extension_filter.py | 2 +- tests/bot/exts/filtering/test_token_filter.py | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/bot/exts/filtering/_filters/filter.py b/bot/exts/filtering/_filters/filter.py index 128e84645e..2b8f8d5d46 100644 --- a/bot/exts/filtering/_filters/filter.py +++ b/bot/exts/filtering/_filters/filter.py @@ -31,9 +31,7 @@ def __init__(self, filter_data: dict, defaults: Defaults | None = None): self.updated_at = arrow.get(filter_data["updated_at"]) self.actions, self.validations = create_settings(filter_data["settings"], defaults=defaults) if self.extra_fields_type: - self.extra_fields = self.extra_fields_type.parse_raw( - filter_data["additional_settings"] or "{}" # noqa: P103 - ) + self.extra_fields = self.extra_fields_type.parse_obj(filter_data["additional_settings"]) else: self.extra_fields = None diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 8fd4ddb132..392428bb06 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -1177,7 +1177,7 @@ async def _post_new_filter( description = description or None payload = { "filter_list": list_id, "content": content, "description": description, - "additional_settings": json.dumps(filter_settings), **settings + "additional_settings": filter_settings, **settings } response = await bot.instance.api_client.post('bot/filter/filters', json=to_serializable(payload)) new_filter = filter_list.add_filter(list_type, response) @@ -1220,7 +1220,7 @@ async def _patch_filter( description = description or None payload = { "filter_list": list_id, "content": content, "description": description, - "additional_settings": json.dumps(filter_settings), **settings + "additional_settings": filter_settings, **settings } response = await bot.instance.api_client.patch( f'bot/filter/filters/{filter_.id}', json=to_serializable(payload) diff --git a/tests/bot/exts/filtering/test_discord_token_filter.py b/tests/bot/exts/filtering/test_discord_token_filter.py index 4d7e69bdc4..a5cddf8d90 100644 --- a/tests/bot/exts/filtering/test_discord_token_filter.py +++ b/tests/bot/exts/filtering/test_discord_token_filter.py @@ -22,7 +22,7 @@ def setUp(self): "content": "discord_token", "description": None, "settings": {}, - "additional_settings": "{}", # noqa: P103 + "additional_settings": {}, "created_at": now, "updated_at": now }) diff --git a/tests/bot/exts/filtering/test_extension_filter.py b/tests/bot/exts/filtering/test_extension_filter.py index 800fad3a07..827d267d29 100644 --- a/tests/bot/exts/filtering/test_extension_filter.py +++ b/tests/bot/exts/filtering/test_extension_filter.py @@ -25,7 +25,7 @@ def setUp(self): for i, filter_content in enumerate(self.whitelist, start=1): filters.append({ "id": i, "content": filter_content, "description": None, "settings": {}, - "additional_settings": "{}", "created_at": now, "updated_at": now # noqa: P103 + "additional_settings": {}, "created_at": now, "updated_at": now # noqa: P103 }) self.filter_list.add_list({ "id": 1, diff --git a/tests/bot/exts/filtering/test_token_filter.py b/tests/bot/exts/filtering/test_token_filter.py index 82cc6b67e8..03fa6b4b9e 100644 --- a/tests/bot/exts/filtering/test_token_filter.py +++ b/tests/bot/exts/filtering/test_token_filter.py @@ -40,7 +40,7 @@ async def test_token_filter_triggers(self): "content": pattern, "description": None, "settings": {}, - "additional_settings": "{}", # noqa: P103 + "additional_settings": {}, "created_at": now, "updated_at": now }) From 88f1d8ccc5f09635a655a66062102670838649f4 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Thu, 6 Apr 2023 01:28:54 +0300 Subject: [PATCH 128/132] Fix settings parser to work when no description and template first --- bot/exts/filtering/_ui/ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 6a9328474f..1690d2286d 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -46,7 +46,7 @@ ALERT_VIEW_TIMEOUT = 3600 SETTINGS_DELIMITER = re.compile(r"\s+(?=\S+=\S+)") -SINGLE_SETTING_PATTERN = re.compile(r"[\w/]+=.+") +SINGLE_SETTING_PATTERN = re.compile(r"(--)?[\w/]+=.+") EDIT_CONFIRMED_MESSAGE = "✅ Edit for `{0}` confirmed" From 543bcdf8a12ed9e61f3116546d91af4e5209379d Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Thu, 6 Apr 2023 01:44:28 +0300 Subject: [PATCH 129/132] Don't notify the user for filters triggered in DMs. --- .../actions/infraction_and_notification.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index a7fb37ca90..818ba3f238 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -141,8 +141,8 @@ class InfractionAndNotification(ActionEntry): "If 0, the infraction will be sent in the context channel. If the ID otherwise fails to resolve, " "it will default to the mod-alerts channel." ), - "dm_content": "The contents of a message to be DMed to the offending user.", - "dm_embed": "The contents of the embed to be DMed to the offending user." + "dm_content": "The contents of a message to be DMed to the offending user. Doesn't send when invoked in DMs.", + "dm_embed": "The contents of the embed to be DMed to the offending user. Doesn't send when invoked in DMs." } dm_content: str @@ -160,8 +160,8 @@ def convert_infraction_name(cls, infr_type: str | Infraction) -> Infraction: return infr_type return Infraction[infr_type.replace(" ", "_").upper()] - async def action(self, ctx: FilterContext) -> None: - """Send the notification to the user, and apply any specified infractions.""" + async def send_message(self, ctx: FilterContext) -> None: + """Send the notification to the user.""" # If there is no infraction to apply, any DM contents already provided in the context take precedence. if self.infraction_type == Infraction.NONE and (ctx.dm_content or ctx.dm_embed): dm_content = ctx.dm_content @@ -184,6 +184,11 @@ async def action(self, ctx: FilterContext) -> None: except Forbidden: ctx.action_descriptions.append("failed to notify") + async def action(self, ctx: FilterContext) -> None: + """Send the notification to the user, and apply any specified infractions.""" + if hasattr(ctx.channel, "category"): # Don't DM the user for filters invoked in DMs. + await self.send_message(ctx) + if self.infraction_type != Infraction.NONE: alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) if self.infraction_channel: From c0f1eb07843c514bd68a9e07d920b822aee5a5fd Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Thu, 6 Apr 2023 14:23:06 +0300 Subject: [PATCH 130/132] Add in_guild attribute to FilterContext --- bot/exts/filtering/_filter_context.py | 8 ++++++-- .../actions/infraction_and_notification.py | 2 +- .../_settings_types/validations/channel_scope.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 483706e2af..05cc23160e 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -6,7 +6,7 @@ from enum import Enum, auto import discord -from discord import DMChannel, Embed, Member, Message, TextChannel, Thread, User +from discord import DMChannel, Embed, Member, Message, StageChannel, TextChannel, Thread, User, VoiceChannel from bot.utils.message_cache import MessageCache @@ -31,7 +31,7 @@ class FilterContext: # Input context event: Event # The type of event author: User | Member | None # Who triggered the event - channel: TextChannel | Thread | DMChannel | None # The channel involved + channel: TextChannel | VoiceChannel | StageChannel | Thread | DMChannel | None # The channel involved content: str | Iterable # What actually needs filtering. The Iterable type depends on the filter list. message: Message | None # The message involved embeds: list[Embed] = field(default_factory=list) # Any embeds involved @@ -57,6 +57,10 @@ class FilterContext: uploaded_attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. upload_deletion_logs: bool = True # Whether it's allowed to upload deletion logs. + def __post_init__(self): + # If it were in the context of a DM there would be a channel. + self.in_guild = self.channel is None or self.channel.guild is not None + @classmethod def from_message( cls, event: Event, message: Message, before: Message | None = None, cache: MessageCache | None = None diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 818ba3f238..508c09c2ac 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -186,7 +186,7 @@ async def send_message(self, ctx: FilterContext) -> None: async def action(self, ctx: FilterContext) -> None: """Send the notification to the user, and apply any specified infractions.""" - if hasattr(ctx.channel, "category"): # Don't DM the user for filters invoked in DMs. + if ctx.in_guild: # Don't DM the user for filters invoked in DMs. await self.send_message(ctx) if self.infraction_type != Infraction.NONE: diff --git a/bot/exts/filtering/_settings_types/validations/channel_scope.py b/bot/exts/filtering/_settings_types/validations/channel_scope.py index 737465137b..45b769d290 100644 --- a/bot/exts/filtering/_settings_types/validations/channel_scope.py +++ b/bot/exts/filtering/_settings_types/validations/channel_scope.py @@ -55,7 +55,7 @@ def triggers_on(self, ctx: FilterContext) -> bool: if not channel: return True - if not hasattr(channel, "category"): # This is not a guild channel, outside the scope of this setting. + if not ctx.in_guild: # This is not a guild channel, outside the scope of this setting. return True if hasattr(channel, "parent"): channel = channel.parent From e0ae45a006bf0f6502e9ae55abb9887785393651 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Thu, 6 Apr 2023 15:14:53 +0300 Subject: [PATCH 131/132] .__origin__ -> get_origin --- bot/exts/filtering/_ui/ui.py | 17 +++++++++-------- bot/exts/filtering/_utils.py | 16 ++++++++++++---- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 1690d2286d..0de511f032 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from enum import EnumMeta from functools import partial -from typing import Any, Callable, Coroutine, Optional, TypeVar +from typing import Any, Callable, Coroutine, Optional, TypeVar, get_origin import discord from discord import Embed, Interaction @@ -21,7 +21,7 @@ from bot.constants import Colours from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._filter_lists import FilterList -from bot.exts.filtering._utils import FakeContext +from bot.exts.filtering._utils import FakeContext, normalize_type from bot.utils.messages import format_channel, format_user, upload_log log = get_logger(__name__) @@ -135,10 +135,11 @@ def populate_embed_from_dict(embed: Embed, data: dict) -> None: def parse_value(value: str, type_: type[T]) -> T: - """Parse the value and attempt to convert it to the provided type.""" - if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias - type_ = type_.__origin__ - if value == '""': + """Parse the value provided in the CLI and attempt to convert it to the provided type.""" + blank = value == '""' + type_ = normalize_type(type_, prioritize_nonetype=blank) + + if blank or isinstance(None, type_): return type_() if type_ in (tuple, list, set): return list(value.split(",")) @@ -461,8 +462,8 @@ async def _prompt_new_value(self, interaction: Interaction, select: discord.ui.S """Prompt the user to give an override value for the setting they selected, and respond to the interaction.""" setting_name = select.values[0] type_ = self.type_per_setting_name[setting_name] - if hasattr(type_, "__origin__"): # In case this is a types.GenericAlias or a typing._GenericAlias - type_ = type_.__origin__ + if origin := get_origin(type_): # In case this is a types.GenericAlias or a typing._GenericAlias + type_ = origin new_view = self.copy() # This is in order to not block the interaction response. There's a potential race condition here, since # a view's method is used without guaranteeing the task completed, but since it depends on user input diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index a43233f20e..97a0fa8d4f 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -132,16 +132,24 @@ def repr_equals(override: Any, default: Any) -> bool: return str(override) == str(default) -def starting_value(type_: type[T]) -> T: - """Return a value of the given type.""" +def normalize_type(type_: type, *, prioritize_nonetype: bool = True) -> type: + """Reduce a given type to one that can be initialized.""" if get_origin(type_) in (Union, types.UnionType): # In case of a Union args = get_args(type_) if type(None) in args: - return None + if prioritize_nonetype: + return type(None) + else: + args = tuple(set(args) - {type(None)}) type_ = args[0] # Pick one, doesn't matter if origin := get_origin(type_): # In case of a parameterized List, Set, Dict etc. - type_ = origin + return origin + return type_ + +def starting_value(type_: type[T]) -> T: + """Return a value of the given type.""" + type_ = normalize_type(type_) try: return type_() except TypeError: # In case it all fails, return a string and let the user handle it. From e9aeb621156e72dda645b3f7d1d9a741ef5699ba Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Thu, 6 Apr 2023 16:05:34 +0300 Subject: [PATCH 132/132] Clarify comment --- bot/exts/filtering/_filter_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 05cc23160e..0794a48e4f 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -58,7 +58,7 @@ class FilterContext: upload_deletion_logs: bool = True # Whether it's allowed to upload deletion logs. def __post_init__(self): - # If it were in the context of a DM there would be a channel. + # If it's in the context of a DM channel, self.channel won't be None, but self.channel.guild will. self.in_guild = self.channel is None or self.channel.guild is not None @classmethod