From b5c541790c038b2c327af204739b5493774d30b3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 00:40:16 -0400 Subject: [PATCH 01/99] feat: rewrite the entire configuration system to use marshmallow and attrs --- app.json | 2 +- modmail/__init__.py | 6 +- modmail/__main__.py | 2 +- modmail/bot.py | 6 +- modmail/config.py | 248 ++++++++++++++++++++-------------- modmail/utils/embeds.py | 4 +- modmail/utils/extensions.py | 4 +- poetry.lock | 257 +++++++++++++++++++++++------------- pyproject.toml | 10 +- requirements.txt | 19 ++- 10 files changed, 350 insertions(+), 208 deletions(-) diff --git a/app.json b/app.json index 11af2756..2a69903a 100644 --- a/app.json +++ b/app.json @@ -1,6 +1,6 @@ { "env": { - "TOKEN": { + "MODMAIL_BOT_TOKEN": { "description": "Discord bot token. This is from https://discord.com/developers/applications", "required": true } diff --git a/modmail/__init__.py b/modmail/__init__.py index 9377ffd2..8f61401e 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -3,10 +3,14 @@ from pathlib import Path import coloredlogs +import environs from modmail.log import ModmailLogger +env = environs.Env() +env.read_env(".env") + logging.TRACE = 5 logging.NOTICE = 25 logging.addLevelName(logging.TRACE, "TRACE") @@ -17,7 +21,7 @@ # this logging level is set to logging.TRACE because if it is not set to the lowest level, # the child level will be limited to the lowest level this is set to. -ROOT_LOG_LEVEL = logging.TRACE +ROOT_LOG_LEVEL = env.log_level("MODMAIL_LOG_LEVEL", logging.TRACE) FMT = "%(asctime)s %(levelname)10s %(name)15s - [%(lineno)5d]: %(message)s" DATEFMT = "%Y/%m/%d %H:%M:%S" diff --git a/modmail/__main__.py b/modmail/__main__.py index fd10af2a..6c1945f5 100644 --- a/modmail/__main__.py +++ b/modmail/__main__.py @@ -21,7 +21,7 @@ def main() -> None: """Run the bot.""" patch_embed() bot = ModmailBot() - bot.run(bot.config.bot.token) + bot.run(bot.config.user.bot.token) if __name__ == "__main__": diff --git a/modmail/bot.py b/modmail/bot.py index e62f1bda..8d30426f 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -11,7 +11,7 @@ from discord.client import _cleanup_loop from discord.ext import commands -from modmail.config import CONFIG +from modmail.config import config from modmail.log import ModmailLogger from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions from modmail.utils.plugins import PLUGINS, walk_plugins @@ -37,7 +37,7 @@ class ModmailBot(commands.Bot): logger: ModmailLogger = logging.getLogger(__name__) def __init__(self, **kwargs): - self.config = CONFIG + self.config = config self.start_time: t.Optional[arrow.Arrow] = None # arrow.utcnow() self.http_session: t.Optional[ClientSession] = None @@ -45,7 +45,7 @@ def __init__(self, **kwargs): activity = Activity(type=discord.ActivityType.listening, name="users dming me!") # listen to messages mentioning the bot or matching the prefix # ! NOTE: This needs to use the configuration system to get the prefix from the db once it exists. - prefix = commands.when_mentioned_or(CONFIG.bot.prefix) + prefix = commands.when_mentioned_or(self.config.user.bot.prefix) # allow only user mentions by default. # ! NOTE: This may change in the future to allow roles as well allowed_mentions = AllowedMentions(everyone=False, users=True, roles=False, replied_user=True) diff --git a/modmail/config.py b/modmail/config.py index dce1efeb..9dbd1396 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,144 +1,194 @@ -import asyncio -import datetime -import json +import inspect import logging import os +import pathlib import sys import typing -from pathlib import Path -from typing import Any, Dict, Optional, Tuple +from collections import defaultdict +from pprint import pprint +import atoml +import attr +import desert import discord -import toml -from discord.ext.commands import BadArgument -from pydantic import BaseModel -from pydantic import BaseSettings as PydanticBaseSettings -from pydantic import Field, SecretStr -from pydantic.color import Color as ColorBase -from pydantic.env_settings import SettingsSourceCallable -from pydantic.types import conint +import environs +import marshmallow +import marshmallow.fields +import marshmallow.validate -log = logging.getLogger(__name__) +ENV_PREFIX = "MODMAIL_" -CONFIG_PATHS: list = [ - f"{os.getcwd()}/config.toml", - f"{os.getcwd()}/modmail/config.toml", - "./config.toml", -] +env = environs.Env(eager=False, expand_vars=True) -DEFAULT_CONFIG_PATHS = [os.path.join(os.path.dirname(__file__), "config-default.toml")] +DEFAULT_CONFIG_PATH = pathlib.Path(os.path.join(os.path.dirname(__file__), "test.toml")) -def determine_file_path( - paths=typing.Union[list, tuple], config_type: str = "default" -) -> typing.Union[str, None]: - path = None - for file_path in paths: - config_file = Path(file_path) - if (config_file).exists(): - path = config_file - log.debug(f"Found {config_type} config at {file_path}") - break - return path or None +def _generate_default_dict(): + """For defaultdicts to default to a defaultdict.""" + return defaultdict(_generate_default_dict) -DEFAULT_CONFIG_PATH = determine_file_path(DEFAULT_CONFIG_PATHS) -USER_CONFIG_PATH = determine_file_path(CONFIG_PATHS, config_type="") +try: + with open(DEFAULT_CONFIG_PATH) as f: + unparsed_user_provided_cfg = defaultdict(_generate_default_dict, atoml.parse(f.read()).value) +except FileNotFoundError: + unparsed_user_provided_cfg = defaultdict(_generate_default_dict) -def toml_default_config_source(settings: PydanticBaseSettings) -> Dict[str, Any]: - """ - A simple settings source that loads variables from a toml file - from within the module's source folder. - Here we happen to choose to use the `env_file_encoding` from Config - when reading `config-default.toml` - """ - return dict(**toml.load(DEFAULT_CONFIG_PATH)) +class _ColourField(marshmallow.fields.Field): + """Class to convert a str or int into a color and deseriaze into a string.""" + + def _serialize(self, value: typing.Any, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: + return str(value.value) + + def _deserialize( + self, + value: typing.Any, + attr: typing.Optional[str], + data: typing.Optional[typing.Mapping[str, typing.Any]], + **kwargs, + ) -> str: + if not isinstance(value, discord.Colour): + value = discord.Colour(int(value, 16)) + return value + + +# marshmallow.fields.Colour = _ColourField -def toml_user_config_source(settings: PydanticBaseSettings) -> Dict[str, Any]: +@attr.s(auto_attribs=True, frozen=True) +class Bot: + """Values that are configuration for the bot itself. + + These are metavalues, and are the token, prefix, database bind, basically all of the stuff that needs to + be known BEFORE attempting to log in to the database or discord. """ - A simple settings source that loads variables from a toml file - from within the module's source folder. - Here we happen to choose to use the `env_file_encoding` from Config - when reading `config-default.toml` + token: str = attr.ib( + default=marshmallow.missing, + metadata={ + "required": True, + "load_only": True, + "allow_none": False, + }, + ) + prefix: str = attr.ib( + default="?", + metadata={ + "allow_none": False, + }, + ) + + class Meta: + load_only = ("token",) + partial = True + + +def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: + if isinstance(col, discord.Colour): + return col + if isinstance(col, str): + col = int(col, 16) + return discord.Colour(col) + + +@attr.s(auto_attribs=True) +class BotModeCfg: + production: bool = desert.ib( + marshmallow.fields.Constant(True), default=True, metadata={"dump_default": True, "dump_only": True} + ) + develop: bool = attr.ib(default=False, metadata={"allow_none": False}) + plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) + + +@attr.s(auto_attribs=True) +class Colours: """ - if USER_CONFIG_PATH: - return dict(**toml.load(USER_CONFIG_PATH)) - else: - return dict() + Default colors. + These should only be changed here to change the default colors. + """ -class BaseSettings(PydanticBaseSettings): - class Config: - extra = "ignore" - env_file = ".env" - env_file_encoding = "utf-8" + base_embed_color: discord.Colour = desert.ib( + _ColourField(), default="0x7289DA", converter=convert_to_color + ) - @classmethod - def customise_sources( - cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - return ( - env_settings, - init_settings, - file_secret_settings, - toml_user_config_source, - toml_default_config_source, - ) +@attr.s(auto_attribs=True) +class DevCfg: + mode: BotModeCfg = BotModeCfg() + log_level: int = desert.ib( + marshmallow.fields.Integer( + validate=marshmallow.validate.Range(0, 50, error="Logging level must be within 0 to 50.") + ), + default=logging.INFO, + ) -class BotConfig(BaseSettings): - prefix: str = "?" - token: str = None - class Config: - # env_prefix = "bot." - allow_mutation = False +@attr.s(auto_attribs=True) +class Cfg: + bot: Bot = Bot() + colours: Colours = Colours() + dev: DevCfg = DevCfg() + class Meta: + exclude = ("bot.token",) -class BotMode(BaseSettings): + +# find and build a bot class from our env +def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing.Dict = None) -> Bot: """ - Bot mode. + Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. - Used to determine when the bot will run. + If defaults is provided, uses a value from there if the environment variable is not set or is None. """ + # get the attributes of the provided class + if defaults is None: + defaults = defaultdict(lambda: marshmallow.missing) + else: + defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) + attribs: typing.Set[attr.Attribute] = set() + for a in dir(klass.__attrs_attrs__): + if hasattr(klass.__attrs_attrs__, a): + if isinstance(attribute := getattr(klass.__attrs_attrs__, a), attr.Attribute): + attribs.add(attribute) + # read the env vars from the above + with env.prefixed(ENV_PREFIX): + kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing + for var in attribs: + kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) + if defaults and kw[var.name] is None: + kw[var.name] = defaults[var.name] - production: bool = True - plugin_dev: bool = False - develop: bool = False + return klass(**kw) -class Colors(BaseSettings): - """ - Default colors. +unparsed_user_provided_cfg["bot"] = attr.asdict( + _build_bot_class(Bot, "BOT_", unparsed_user_provided_cfg["bot"]) +) - These should only be changed here to change the default colors. - """ +# env.seal() - embed_color: ColorBase = "0087BD" +# build configuration +Configuration = desert.schema_class(Cfg) # noqa: N818 +USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.RAISE) -class DevConfig(BaseSettings): - """ - Developer specific configuration. - These settings should not be changed unless you know what you're doing. - """ +# hide the bot token from serilzation +# this prevents the token from being saved to places. + + +DEFAULTS_CONFIG = Cfg() - log_level: conint(ge=0, le=50) = getattr(logging, "NOTICE", 25) - mode: BotMode +@attr.s(auto_attribs=True, slots=True) +class Config: + user: Cfg + default: Cfg -class ModmailConfig(BaseSettings): - bot: BotConfig - dev: DevConfig - colors: Colors +config = Config(USER_PROVIDED_CONFIG, DEFAULTS_CONFIG) -CONFIG = ModmailConfig() +print(config.user.bot.prefix) diff --git a/modmail/utils/embeds.py b/modmail/utils/embeds.py index 6e0bb150..1e945c4c 100644 --- a/modmail/utils/embeds.py +++ b/modmail/utils/embeds.py @@ -3,10 +3,10 @@ import discord from discord.embeds import EmptyEmbed -from modmail.config import CONFIG +from modmail.config import config -DEFAULT_COLOR = int(CONFIG.colors.embed_color.as_hex().lstrip("#"), 16) +DEFAULT_COLOR = config.user.colours.base_embed_color original_init = discord.Embed.__init__ diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index be547c50..23f10880 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -8,7 +8,7 @@ import typing as t from modmail import extensions -from modmail.config import CONFIG +from modmail.config import config from modmail.log import ModmailLogger from modmail.utils.cogs import BOT_MODES, BotModes, ExtMetadata @@ -35,7 +35,7 @@ def determine_bot_mode() -> int: """ bot_mode = 0 for mode in BotModes: - if getattr(CONFIG.dev.mode, unqualify(str(mode)).lower(), True): + if getattr(config.user.dev.mode, unqualify(str(mode)).lower(), True): bot_mode += mode.value return bot_mode diff --git a/poetry.lock b/poetry.lock index d0fdf060..86aa99a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -58,6 +58,14 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "atoml" +version = "1.0.3" +description = "Yet another style preserving TOML library" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "attrs" version = "21.2.0" @@ -101,7 +109,7 @@ stevedore = ">=1.20.0" [[package]] name = "black" -version = "21.8b0" +version = "21.9b0" description = "The uncompromising code formatter." category = "dev" optional = false @@ -182,7 +190,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "charset-normalizer" -version = "2.0.4" +version = "2.0.6" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "dev" optional = false @@ -250,6 +258,23 @@ toml = {version = "*", optional = true, markers = "extra == \"toml\""} [package.extras] toml = ["toml"] +[[package]] +name = "desert" +version = "2020.11.18" +description = "Deserialize to objects while staying DRY" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +attrs = "*" +marshmallow = ">=3.0" +typing-inspect = "*" + +[package.extras] +dev = ["coverage", "cuvner", "marshmallow-enum", "marshmallow-union", "pytest", "pytest-cov", "pytest-sphinx", "pytest-travis-fold", "tox", "importlib-metadata", "versioneer", "black", "pylint", "pex", "bump2version", "docutils", "check-manifest", "readme-renderer", "pygments", "isort", "mypy", "towncrier", "twine", "wheel"] +test = ["coverage", "cuvner", "marshmallow-enum", "marshmallow-union", "pytest", "pytest-cov", "pytest-sphinx", "pytest-travis-fold", "tox", "importlib-metadata"] + [[package]] name = "discord.py" version = "2.0.0a0" @@ -269,6 +294,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] [package.source] type = "url" url = "https://github.com/Rapptz/discord.py/archive/master.zip" + [[package]] name = "distlib" version = "0.3.2" @@ -277,6 +303,24 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "environs" +version = "9.3.3" +description = "simplified environment variable parsing" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +marshmallow = ">=3.0.0" +python-dotenv = "*" + +[package.extras] +dev = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url", "flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)", "tox"] +django = ["dj-database-url", "dj-email-url", "django-cache-url"] +lint = ["flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)"] +tests = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url"] + [[package]] name = "execnet" version = "1.9.0" @@ -448,11 +492,11 @@ smmap = ">=3.0.1,<5" [[package]] name = "gitpython" -version = "3.1.20" -description = "Python Git Library" +version = "3.1.24" +description = "GitPython is a python library used to interact with Git repositories" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" @@ -460,18 +504,18 @@ typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\" [[package]] name = "humanfriendly" -version = "9.2" +version = "10.0" description = "Human friendly output for text interfaces using Python" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -pyreadline = {version = "*", markers = "sys_platform == \"win32\""} +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} [[package]] name = "identify" -version = "2.2.13" +version = "2.2.14" description = "File identification library for Python" category = "dev" optional = false @@ -559,6 +603,31 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "marshmallow" +version = "3.13.0" +description = "A lightweight library for converting complex datatypes to and from native Python datatypes." +category = "main" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["pytest", "pytz", "simplejson", "mypy (==0.910)", "flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "pre-commit (>=2.4,<3.0)", "tox"] +docs = ["sphinx (==4.1.1)", "sphinx-issues (==1.2.0)", "alabaster (==0.7.12)", "sphinx-version-warning (==1.1.2)", "autodocsumm (==0.2.6)"] +lint = ["mypy (==0.910)", "flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "pre-commit (>=2.4,<3.0)"] +tests = ["pytest", "pytz", "simplejson"] + +[[package]] +name = "marshmallow-enum" +version = "1.5.1" +description = "Enum field for Marshmallow" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +marshmallow = ">=2.0.0" + [[package]] name = "mccabe" version = "0.6.1" @@ -627,14 +696,11 @@ pymdown-extensions = ">=7.0" [[package]] name = "mkdocs-material-extensions" -version = "1.0.1" +version = "1.0.3" description = "Extension pack for Python Markdown." category = "dev" optional = false -python-versions = ">=3.5" - -[package.dependencies] -mkdocs-material = ">=5.0.0" +python-versions = ">=3.6" [[package]] name = "mslex" @@ -656,7 +722,7 @@ python-versions = ">=3.6" name = "mypy-extensions" version = "0.4.3" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -733,7 +799,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "2.14.1" +version = "2.15.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false @@ -796,22 +862,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "pydantic" -version = "1.8.2" -description = "Data validation and settings management using python 3.6 type hinting" -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -python-dotenv = {version = ">=0.10.4", optional = true, markers = "extra == \"dotenv\""} -typing-extensions = ">=3.7.4.3" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - [[package]] name = "pydocstyle" version = "6.1.1" @@ -862,9 +912,9 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] -name = "pyreadline" -version = "2.1" -description = "A python implmementation of GNU readline." +name = "pyreadline3" +version = "3.3" +description = "A python implementation of GNU readline." category = "main" optional = false python-versions = "*" @@ -1078,7 +1128,7 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" [[package]] name = "taskipy" -version = "1.8.1" +version = "1.8.2" description = "tasks runner for python projects" category = "dev" optional = false @@ -1086,7 +1136,7 @@ python-versions = ">=3.6,<4.0" [package.dependencies] colorama = ">=0.4.4,<0.5.0" -mslex = ">=0.3.0,<0.4.0" +mslex = {version = ">=0.3.0,<0.4.0", markers = "sys_platform == \"win32\""} psutil = ">=5.7.2,<6.0.0" toml = ">=0.10.0,<0.11.0" @@ -1115,7 +1165,7 @@ test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybi name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" +category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" @@ -1135,6 +1185,18 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "typing-inspect" +version = "0.7.1" +description = "Runtime inspection utilities for typing module." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +mypy-extensions = ">=0.3.0" +typing-extensions = ">=3.7.4" + [[package]] name = "urllib3" version = "1.26.6" @@ -1150,7 +1212,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.7.2" +version = "20.8.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -1205,7 +1267,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "13d426f2143ed7a3fb374ff734603e7dd6e16bf259834232f9fb6c056f11f2bb" +content-hash = "6cfec48a5437401e5e5aeece5efc90f48eae9e2903d6117c4f2cc850af33cdab" [metadata.files] aiodns = [ @@ -1263,6 +1325,10 @@ atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] +atoml = [ + {file = "atoml-1.0.3-py3-none-any.whl", hash = "sha256:944c0e9043ca4e0729d4125132841ef1110677b8d015a624892d63cdc4988655"}, + {file = "atoml-1.0.3.tar.gz", hash = "sha256:5dd70efcafde94a6aa5db2e8c6af5d832bf95b38f47d3283ee3779e920218e94"}, +] attrs = [ {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, @@ -1276,8 +1342,8 @@ bandit = [ {file = "bandit-1.7.0.tar.gz", hash = "sha256:8a4c7415254d75df8ff3c3b15cfe9042ecee628a1e40b44c15a98890fbfc2608"}, ] black = [ - {file = "black-21.8b0-py3-none-any.whl", hash = "sha256:2a0f9a8c2b2a60dbcf1ccb058842fb22bdbbcb2f32c6cc02d9578f90b92ce8b7"}, - {file = "black-21.8b0.tar.gz", hash = "sha256:570608d28aa3af1792b98c4a337dbac6367877b47b12b88ab42095cfc1a627c2"}, + {file = "black-21.9b0-py3-none-any.whl", hash = "sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115"}, + {file = "black-21.9b0.tar.gz", hash = "sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91"}, ] brotlipy = [ {file = "brotlipy-0.7.0-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2"}, @@ -1361,11 +1427,6 @@ cffi = [ {file = "cffi-1.14.6-cp27-cp27m-win_amd64.whl", hash = "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33"}, - {file = "cffi-1.14.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5"}, - {file = "cffi-1.14.6-cp35-cp35m-win32.whl", hash = "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca"}, - {file = "cffi-1.14.6-cp35-cp35m-win_amd64.whl", hash = "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218"}, {file = "cffi-1.14.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195"}, @@ -1409,8 +1470,8 @@ chardet = [ {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, - {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, + {file = "charset-normalizer-2.0.6.tar.gz", hash = "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"}, + {file = "charset_normalizer-2.0.6-py3-none-any.whl", hash = "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6"}, ] click = [ {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, @@ -1483,11 +1544,19 @@ coverage = [ {file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"}, {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] +desert = [ + {file = "desert-2020.11.18-py3-none-any.whl", hash = "sha256:6392702be7952fb9c8bbc775425fa929c1eab5c06552c2890e82a24964eeb084"}, + {file = "desert-2020.11.18.tar.gz", hash = "sha256:d7b7fb521dc84eec955a766ed7e37349f998cf047f37fd9596cb09737d63c62d"}, +] "discord.py" = [] distlib = [ {file = "distlib-0.3.2-py2.py3-none-any.whl", hash = "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c"}, {file = "distlib-0.3.2.zip", hash = "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736"}, ] +environs = [ + {file = "environs-9.3.3-py2.py3-none-any.whl", hash = "sha256:ee5466156b50fe03aa9fec6e720feea577b5bf515d7f21b2c46608272557ba26"}, + {file = "environs-9.3.3.tar.gz", hash = "sha256:72b867ff7b553076cdd90f3ee01ecc1cf854987639c9c459f0ed0d3d44ae490c"}, +] execnet = [ {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, @@ -1542,16 +1611,16 @@ gitdb = [ {file = "gitdb-4.0.7.tar.gz", hash = "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"}, ] gitpython = [ - {file = "GitPython-3.1.20-py3-none-any.whl", hash = "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a"}, - {file = "GitPython-3.1.20.tar.gz", hash = "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519"}, + {file = "GitPython-3.1.24-py3-none-any.whl", hash = "sha256:dc0a7f2f697657acc8d7f89033e8b1ea94dd90356b2983bca89dc8d2ab3cc647"}, + {file = "GitPython-3.1.24.tar.gz", hash = "sha256:df83fdf5e684fef7c6ee2c02fc68a5ceb7e7e759d08b694088d0cacb4eba59e5"}, ] humanfriendly = [ - {file = "humanfriendly-9.2-py2.py3-none-any.whl", hash = "sha256:332da98c24cc150efcc91b5508b19115209272bfdf4b0764a56795932f854271"}, - {file = "humanfriendly-9.2.tar.gz", hash = "sha256:f7dba53ac7935fd0b4a2fc9a29e316ddd9ea135fb3052d3d0279d10c18ff9c48"}, + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, ] identify = [ - {file = "identify-2.2.13-py2.py3-none-any.whl", hash = "sha256:7199679b5be13a6b40e6e19ea473e789b11b4e3b60986499b1f589ffb03c217c"}, - {file = "identify-2.2.13.tar.gz", hash = "sha256:7bc6e829392bd017236531963d2d937d66fc27cadc643ac0aba2ce9f26157c79"}, + {file = "identify-2.2.14-py2.py3-none-any.whl", hash = "sha256:113a76a6ba614d2a3dd408b3504446bcfac0370da5995aa6a17fd7c6dffde02d"}, + {file = "identify-2.2.14.tar.gz", hash = "sha256:32f465f3c48083f345ad29a9df8419a4ce0674bf4a8c3245191d65c83634bdbf"}, ] idna = [ {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, @@ -1578,12 +1647,22 @@ markdown = [ {file = "Markdown-3.3.4.tar.gz", hash = "sha256:31b5b491868dcc87d6c24b7e3d19a0d730d59d3e46f4eea6430a321bed387a49"}, ] markupsafe = [ + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, @@ -1592,14 +1671,21 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, @@ -1609,10 +1695,21 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] +marshmallow = [ + {file = "marshmallow-3.13.0-py2.py3-none-any.whl", hash = "sha256:dd4724335d3c2b870b641ffe4a2f8728a1380cd2e7e2312756715ffeaa82b842"}, + {file = "marshmallow-3.13.0.tar.gz", hash = "sha256:c67929438fd73a2be92128caa0325b1b5ed8b626d91a094d2f7f2771bf1f1c0e"}, +] +marshmallow-enum = [ + {file = "marshmallow-enum-1.5.1.tar.gz", hash = "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58"}, + {file = "marshmallow_enum-1.5.1-py2.py3-none-any.whl", hash = "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072"}, +] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, @@ -1634,8 +1731,8 @@ mkdocs-material = [ {file = "mkdocs_material-7.2.6-py2.py3-none-any.whl", hash = "sha256:4c6939b9d7d5c6db948ab02df8525c64211828ddf33286acea8b9d2115cec369"}, ] mkdocs-material-extensions = [ - {file = "mkdocs-material-extensions-1.0.1.tar.gz", hash = "sha256:6947fb7f5e4291e3c61405bad3539d81e0b3cd62ae0d66ced018128af509c68f"}, - {file = "mkdocs_material_extensions-1.0.1-py3-none-any.whl", hash = "sha256:d90c807a88348aa6d1805657ec5c0b2d8d609c110e62b9dce4daf7fa981fa338"}, + {file = "mkdocs-material-extensions-1.0.3.tar.gz", hash = "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2"}, + {file = "mkdocs_material_extensions-1.0.3-py3-none-any.whl", hash = "sha256:a82b70e533ce060b2a5d9eb2bc2e1be201cf61f901f93704b4acf6e3d5983a44"}, ] mslex = [ {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, @@ -1713,8 +1810,8 @@ pluggy = [ {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] pre-commit = [ - {file = "pre_commit-2.14.1-py2.py3-none-any.whl", hash = "sha256:a22d12a02da4d8df314187dfe7a61bda6291d57992060522feed30c8cd658b68"}, - {file = "pre_commit-2.14.1.tar.gz", hash = "sha256:7977a3103927932d4823178cbe4719ab55bb336f42a9f3bb2776cff99007a117"}, + {file = "pre_commit-2.15.0-py2.py3-none-any.whl", hash = "sha256:a4ed01000afcb484d9eb8d504272e642c4c4099bbad3a6b27e519bd6a3e928a6"}, + {file = "pre_commit-2.15.0.tar.gz", hash = "sha256:3c25add78dbdfb6a28a651780d5c311ac40dd17f160eb3954a0c59da40a505a7"}, ] psutil = [ {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, @@ -1793,30 +1890,6 @@ pycparser = [ {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, ] -pydantic = [ - {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, - {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, - {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, - {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, - {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, - {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, - {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, - {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, - {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, - {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, -] pydocstyle = [ {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, {file = "pydocstyle-6.1.1.tar.gz", hash = "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc"}, @@ -1837,10 +1910,9 @@ pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] -pyreadline = [ - {file = "pyreadline-2.1.win-amd64.exe", hash = "sha256:9ce5fa65b8992dfa373bddc5b6e0864ead8f291c94fbfec05fbd5c836162e67b"}, - {file = "pyreadline-2.1.win32.exe", hash = "sha256:65540c21bfe14405a3a77e4c085ecfce88724743a4ead47c66b84defcf82c32e"}, - {file = "pyreadline-2.1.zip", hash = "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1"}, +pyreadline3 = [ + {file = "pyreadline3-3.3-py3-none-any.whl", hash = "sha256:0003fd0079d152ecbd8111202c5a7dfa6a5569ffd65b235e45f3c2ecbee337b4"}, + {file = "pyreadline3-3.3.tar.gz", hash = "sha256:ff3b5a1ac0010d0967869f723e687d42cabc7dccf33b14934c92aa5168d260b3"}, ] pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, @@ -1975,8 +2047,8 @@ stevedore = [ {file = "stevedore-3.4.0.tar.gz", hash = "sha256:59b58edb7f57b11897f150475e7bc0c39c5381f0b8e3fa9f5c20ce6c89ec4aa1"}, ] taskipy = [ - {file = "taskipy-1.8.1-py3-none-any.whl", hash = "sha256:2b98f499966e40175d1f1306a64587f49dfa41b90d0d86c8f28b067cc58d0a56"}, - {file = "taskipy-1.8.1.tar.gz", hash = "sha256:7a2404125817e45d80e13fa663cae35da6e8ba590230094e815633653e25f98f"}, + {file = "taskipy-1.8.2-py3-none-any.whl", hash = "sha256:24b899ae17908fe9a61f4dc596792d5d2afef471ecfd8c6e9abb68a9568b40b7"}, + {file = "taskipy-1.8.2.tar.gz", hash = "sha256:36e958f646f2c435b39f748b8bbdb0b9c33a2c4a96b293427feaf48ad4e2caa0"}, ] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, @@ -1998,13 +2070,18 @@ typing-extensions = [ {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, ] +typing-inspect = [ + {file = "typing_inspect-0.7.1-py2-none-any.whl", hash = "sha256:b1f56c0783ef0f25fb064a01be6e5407e54cf4a4bf4f3ba3fe51e0bd6dcea9e5"}, + {file = "typing_inspect-0.7.1-py3-none-any.whl", hash = "sha256:3cd7d4563e997719a710a3bfe7ffb544c6b72069b6812a02e9b414a8fa3aaa6b"}, + {file = "typing_inspect-0.7.1.tar.gz", hash = "sha256:047d4097d9b17f46531bf6f014356111a1b6fb821a24fe7ac909853ca2a782aa"}, +] urllib3 = [ {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, ] virtualenv = [ - {file = "virtualenv-20.7.2-py2.py3-none-any.whl", hash = "sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06"}, - {file = "virtualenv-20.7.2.tar.gz", hash = "sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0"}, + {file = "virtualenv-20.8.0-py2.py3-none-any.whl", hash = "sha256:a4b987ec31c3c9996cf1bc865332f967fe4a0512c41b39652d6224f696e69da5"}, + {file = "virtualenv-20.8.0.tar.gz", hash = "sha256:4da4ac43888e97de9cf4fdd870f48ed864bbfd133d2c46cbdec941fed4a25aef"}, ] watchdog = [ {file = "watchdog-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5f57ce4f7e498278fb2a091f39359930144a0f2f90ea8cbf4523c4e25de34028"}, diff --git a/pyproject.toml b/pyproject.toml index e809f636..dad595b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,14 @@ arrow = "^1.1.1" colorama = "^0.4.3" coloredlogs = "^15.0" "discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/master.zip" } -pydantic = { version = "^1.8.2", extras = ["dotenv"] } -toml = "^0.10.2" +atoml = "^1.0.3" +attrs = "^21.2.0" +desert = "^2020.11.18" +environs = "~=9.3.3" +marshmallow = "~=3.13.0" +python-dotenv = "^0.19.0" +typing-extensions = "^3.10.0.2" +marshmallow-enum = "^1.5.1" [tool.poetry.extras] diff --git a/requirements.txt b/requirements.txt index 6ed382e0..3ffd847d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,24 +5,29 @@ aiodns==3.0.0; python_version >= "3.6" or python_version >= "3.6" and python_ful aiohttp==3.7.4.post0; python_version >= "3.6" arrow==1.1.1; python_version >= "3.6" async-timeout==3.0.1; python_full_version >= "3.5.3" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" +atoml==1.0.3; python_version >= "3.6" +attrs==21.2.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") brotlipy==0.7.0; python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" cchardet==2.1.7; python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" cffi==1.14.6; python_version >= "3.6" chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") coloredlogs==15.0.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +desert==2020.11.18; python_version >= "3.6" discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_full_version >= "3.8.0" -humanfriendly==9.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" +environs==9.3.3; python_version >= "3.6" +humanfriendly==10.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" idna==3.2; python_version >= "3.6" +marshmallow-enum==1.5.1 +marshmallow==3.13.0; python_version >= "3.5" multidict==5.1.0; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" +mypy-extensions==0.4.3; python_version >= "3.6" pycares==4.0.0; python_version >= "3.6" pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" -pydantic==1.8.2; python_full_version >= "3.6.1" -pyreadline==2.1; python_version >= "2.7" and python_full_version < "3.0.0" and sys_platform == "win32" or python_full_version >= "3.5.0" and sys_platform == "win32" +pyreadline3==3.3; sys_platform == "win32" and python_version >= "3.8" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0") python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -python-dotenv==0.19.0; python_full_version >= "3.6.1" and python_version >= "3.5" +python-dotenv==0.19.0; python_version >= "3.5" six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") -typing-extensions==3.10.0.0; python_version >= "3.6" or python_full_version >= "3.6.1" or python_version >= "3.6" and python_full_version >= "3.8.0" +typing-extensions==3.10.0.2 +typing-inspect==0.7.1; python_version >= "3.6" yarl==1.6.3; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" From 6a1b0c26a6ffab07cd97db5f48ec6e4b4d7ef310 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 01:51:32 -0400 Subject: [PATCH 02/99] config: remove config_default.toml since its now part of the classes Signed-off-by: onerandomusername --- modmail/config-default.toml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 modmail/config-default.toml diff --git a/modmail/config-default.toml b/modmail/config-default.toml deleted file mode 100644 index 29f7cfc5..00000000 --- a/modmail/config-default.toml +++ /dev/null @@ -1,10 +0,0 @@ -[bot] -prefix = "?" - -[colors] - -[dev] -log_level = 25 # "NOTICE" - -[dev.mode] -production = true From 5afc19691eb723b3cf1d96b84c63fd94b1cfe1d5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 01:52:43 -0400 Subject: [PATCH 03/99] fix: add missing items to the configuration Signed-off-by: onerandomusername --- modmail/__init__.py | 2 +- modmail/config.py | 128 +++++++++++++++++++++++++++++++------------- tox.ini | 1 - 3 files changed, 91 insertions(+), 40 deletions(-) diff --git a/modmail/__init__.py b/modmail/__init__.py index 8f61401e..0d07a6e8 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -9,7 +9,7 @@ env = environs.Env() -env.read_env(".env") +env.read_env(".env", recurse=False) logging.TRACE = 5 logging.NOTICE = 25 diff --git a/modmail/config.py b/modmail/config.py index 9dbd1396..b3f8e096 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -2,44 +2,75 @@ import logging import os import pathlib -import sys import typing from collections import defaultdict -from pprint import pprint import atoml import attr import desert import discord +import discord.ext.commands.converter import environs import marshmallow import marshmallow.fields import marshmallow.validate +_CWD = pathlib.Path(os.getcwd()) ENV_PREFIX = "MODMAIL_" +USER_CONFIG_TOML = _CWD / "modmail_config.toml" env = environs.Env(eager=False, expand_vars=True) +env.read_env(_CWD / ".env", recurse=False) -DEFAULT_CONFIG_PATH = pathlib.Path(os.path.join(os.path.dirname(__file__), "test.toml")) - - -def _generate_default_dict(): +def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" return defaultdict(_generate_default_dict) +unparsed_user_provided_cfg = defaultdict(lambda: marshmallow.missing) try: - with open(DEFAULT_CONFIG_PATH) as f: - unparsed_user_provided_cfg = defaultdict(_generate_default_dict, atoml.parse(f.read()).value) + with open(USER_CONFIG_TOML) as f: + unparsed_user_provided_cfg.update(atoml.parse(f.read()).value) except FileNotFoundError: - unparsed_user_provided_cfg = defaultdict(_generate_default_dict) + pass + + +def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: + """Convert a string or integer to a discord.Colour. Also supports being passed a discord.Colour.""" + if isinstance(col, discord.Colour): + return col + if isinstance(col, str): + col = int(col, 16) + return discord.Colour(col) class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deseriaze into a string.""" + class ColourConvert(discord.ext.commands.converter.ColourConverter): + def convert(self, argument: str) -> discord.Colour: + if argument[0] == "#": + return self.parse_hex_number(argument[1:]) + + if argument[0:2] == "0x": + rest = argument[2:] + # Legacy backwards compatible syntax + if rest.startswith("#"): + return self.parse_hex_number(rest[1:]) + return self.parse_hex_number(rest) + + arg = argument.lower() + if arg[0:3] == "rgb": + return self.parse_rgb(arg) + + arg = arg.replace(" ", "_") + method = getattr(discord.Colour, arg, None) + if arg.startswith("from_") or method is None or not inspect.ismethod(method): + raise discord.ext.commands.converter.BadColourArgument(arg) + return method() + def _serialize(self, value: typing.Any, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: return str(value.value) @@ -51,16 +82,17 @@ def _deserialize( **kwargs, ) -> str: if not isinstance(value, discord.Colour): - value = discord.Colour(int(value, 16)) + if isinstance(value, str): + value = self.ColourConvert().convert(value) + else: + value = discord.Colour(value) return value -# marshmallow.fields.Colour = _ColourField - - -@attr.s(auto_attribs=True, frozen=True) +@attr.s(auto_attribs=True) class Bot: - """Values that are configuration for the bot itself. + """ + Values that are configuration for the bot itself. These are metavalues, and are the token, prefix, database bind, basically all of the stuff that needs to be known BEFORE attempting to log in to the database or discord. @@ -68,6 +100,7 @@ class Bot: token: str = attr.ib( default=marshmallow.missing, + on_setattr=attr.setters.NO_OP, metadata={ "required": True, "load_only": True, @@ -76,28 +109,34 @@ class Bot: ) prefix: str = attr.ib( default="?", + converter=lambda x: "?" if x is None else x, metadata={ "allow_none": False, }, ) class Meta: + """Marshmallow configuration class for schema generation.""" + load_only = ("token",) partial = True -def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: - if isinstance(col, discord.Colour): - return col - if isinstance(col, str): - col = int(col, 16) - return discord.Colour(col) +@attr.s(auto_attribs=True, frozen=True) +class BotModeCfg: + """ + The three bot modes for the bot. Enabling some of these may enable other bot features. + `production` is used internally and is always True. + `develop` enables additonal features which are useful for bot developers. + `plugin_dev` enables additional commands which are useful when working with plugins. + """ -@attr.s(auto_attribs=True) -class BotModeCfg: production: bool = desert.ib( - marshmallow.fields.Constant(True), default=True, metadata={"dump_default": True, "dump_only": True} + marshmallow.fields.Constant(True), + default=True, + converter=lambda _: True, + metadata={"dump_default": True, "dump_only": True}, ) develop: bool = attr.ib(default=False, metadata={"allow_none": False}) plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) @@ -118,6 +157,8 @@ class Colours: @attr.s(auto_attribs=True) class DevCfg: + """Developer configuration. These values should not be changed unless you know what you're doing.""" + mode: BotModeCfg = BotModeCfg() log_level: int = desert.ib( marshmallow.fields.Integer( @@ -129,11 +170,20 @@ class DevCfg: @attr.s(auto_attribs=True) class Cfg: + """ + Base configuration attrs class. + + The reason this creates defaults of the variables is so + we can get a clean default variable if we don't pass anything. + """ + bot: Bot = Bot() colours: Colours = Colours() dev: DevCfg = DevCfg() class Meta: + """Marshmallow configuration class for schema generation.""" + exclude = ("bot.token",) @@ -165,30 +215,32 @@ def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing return klass(**kw) +_toml_bot_cfg = unparsed_user_provided_cfg["bot"] unparsed_user_provided_cfg["bot"] = attr.asdict( - _build_bot_class(Bot, "BOT_", unparsed_user_provided_cfg["bot"]) + _build_bot_class(Bot, "BOT_", _toml_bot_cfg if _toml_bot_cfg is not marshmallow.missing else None) ) - -# env.seal() - +del _toml_bot_cfg # build configuration Configuration = desert.schema_class(Cfg) # noqa: N818 -USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.RAISE) - +USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.EXCLUDE) +DEFAULTS_CONFIG = Cfg() -# hide the bot token from serilzation -# this prevents the token from being saved to places. +@attr.s(auto_attribs=True, slots=True, kw_only=True) +class Config: + """ + Base configuration variable. Used across the entire bot for configuration variables. -DEFAULTS_CONFIG = Cfg() + Holds two variables, default and user. + Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + User is a Cfg schema instance, generated from a combination of the defaults, + user provided toml, and environment variables. + """ -@attr.s(auto_attribs=True, slots=True) -class Config: user: Cfg default: Cfg + schema: marshmallow.Schema -config = Config(USER_PROVIDED_CONFIG, DEFAULTS_CONFIG) - -print(config.user.bot.prefix) +config = Config(user=USER_PROVIDED_CONFIG, default=DEFAULTS_CONFIG, schema=Configuration) diff --git a/tox.ini b/tox.ini index bc557a04..405a6e64 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,6 @@ max-line-length=110 application_import_names=modmail docstring-convention=all exclude= - modmail/config.py, __pycache__, .cache,.git, .md,.svg,.png, From f8fabf5f034acb31d6f5986f475ec54de47dac4c Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 03:23:37 -0400 Subject: [PATCH 04/99] fix: have a test token in the pytest env Signed-off-by: onerandomusername --- .github/workflows/lint_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index a9bb18c5..293d1941 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -162,6 +162,8 @@ jobs: # coverage report to github. - name: Run tests and generate coverage report run: pytest -n auto --dist loadfile --cov --disable-warnings -q + env: + MODMAIL_BOT_TOKEN: NjQzOTQ1MjY0ODY4MDk4MDQ5.342b.4inDLBILY69LOLfyi6jk420dpyjoEVsCoModM # This step will publish the coverage reports to codecov.io and # print a "job" link in the output of the GitHub Action From 6cc4e20bce8954c4ff45ea6f66b41e134e1164f7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 13:24:18 -0400 Subject: [PATCH 05/99] enable slots on cfg classes, fix colour converter, properly prevent token from being changed Signed-off-by: onerandomusername --- modmail/config.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index b3f8e096..4fa0938c 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -37,20 +37,16 @@ def _generate_default_dict() -> defaultdict: pass -def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: - """Convert a string or integer to a discord.Colour. Also supports being passed a discord.Colour.""" - if isinstance(col, discord.Colour): - return col - if isinstance(col, str): - col = int(col, 16) - return discord.Colour(col) - - class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deseriaze into a string.""" class ColourConvert(discord.ext.commands.converter.ColourConverter): def convert(self, argument: str) -> discord.Colour: + if isinstance(argument, discord.Colour): + return argument + if not isinstance(argument, str): + argument = str(argument) + if argument[0] == "#": return self.parse_hex_number(argument[1:]) @@ -89,7 +85,12 @@ def _deserialize( return value -@attr.s(auto_attribs=True) +def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: + """Convert a string or integer to a discord.Colour. Also supports being passed a discord.Colour.""" + return _ColourField.ColourConvert().convert(col) + + +@attr.s(auto_attribs=True, slots=True) class Bot: """ Values that are configuration for the bot itself. @@ -100,10 +101,11 @@ class Bot: token: str = attr.ib( default=marshmallow.missing, - on_setattr=attr.setters.NO_OP, + on_setattr=attr.setters.frozen, + repr=False, metadata={ "required": True, - "load_only": True, + "dump_only": True, "allow_none": False, }, ) @@ -122,7 +124,7 @@ class Meta: partial = True -@attr.s(auto_attribs=True, frozen=True) +@attr.s(auto_attribs=True, slots=True, frozen=True) class BotModeCfg: """ The three bot modes for the bot. Enabling some of these may enable other bot features. @@ -142,7 +144,7 @@ class BotModeCfg: plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) -@attr.s(auto_attribs=True) +@attr.s(auto_attribs=True, slots=True) class Colours: """ Default colors. @@ -155,20 +157,21 @@ class Colours: ) -@attr.s(auto_attribs=True) +@attr.s(auto_attribs=True, slots=True) class DevCfg: """Developer configuration. These values should not be changed unless you know what you're doing.""" mode: BotModeCfg = BotModeCfg() - log_level: int = desert.ib( - marshmallow.fields.Integer( - validate=marshmallow.validate.Range(0, 50, error="Logging level must be within 0 to 50.") - ), - default=logging.INFO, - ) + log_level: int = attr.ib(default=logging.INFO) + + @log_level.validator + def _log_level_validator(self, _: attr.Attribute, value: int) -> None: + """Validate that log_level is within 0 to 50.""" + if value not in range(0, 50): + raise ValueError("log_level must be an integer within 0 to 50.") -@attr.s(auto_attribs=True) +@attr.s(auto_attribs=True, slots=True) class Cfg: """ Base configuration attrs class. From 82b1c73221bf55bdacc1ed1245fd3c1d5dd4be60 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 19:26:31 -0400 Subject: [PATCH 06/99] fix: remove unused class definitions when converting to a marshmallow schema, desert does not use class defined Meta classes Signed-off-by: onerandomusername --- modmail/config.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 4fa0938c..f482411f 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -117,12 +117,6 @@ class Bot: }, ) - class Meta: - """Marshmallow configuration class for schema generation.""" - - load_only = ("token",) - partial = True - @attr.s(auto_attribs=True, slots=True, frozen=True) class BotModeCfg: @@ -184,11 +178,6 @@ class Cfg: colours: Colours = Colours() dev: DevCfg = DevCfg() - class Meta: - """Marshmallow configuration class for schema generation.""" - - exclude = ("bot.token",) - # find and build a bot class from our env def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing.Dict = None) -> Bot: @@ -224,7 +213,7 @@ def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing ) del _toml_bot_cfg # build configuration -Configuration = desert.schema_class(Cfg) # noqa: N818 +Configuration = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.EXCLUDE) DEFAULTS_CONFIG = Cfg() From 4a80406ae833eb83f18dc7f8cc5f35570ea0a26e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 19:28:04 -0400 Subject: [PATCH 07/99] config: add autogenerated default cfg toml file Signed-off-by: onerandomusername --- .gitignore | 2 +- modmail/config.py | 31 +++++++++++++++++++++++++++++++ modmail/default_config.toml | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 modmail/default_config.toml diff --git a/.gitignore b/.gitignore index 523110e0..d737f24f 100644 --- a/.gitignore +++ b/.gitignore @@ -136,7 +136,7 @@ dmypy.json logs # Configuration -*config.toml +/modmail_config.toml # Custom docker compose override docker-compose.override.yml diff --git a/modmail/config.py b/modmail/config.py index f482411f..49e88dcc 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -236,3 +236,34 @@ class Config: config = Config(user=USER_PROVIDED_CONFIG, default=DEFAULTS_CONFIG, schema=Configuration) + +if __name__ == "__main__": + # if this file is run directly, export the configuration to a defaults file. + + dump: dict = Configuration().dump(DEFAULTS_CONFIG) + + # Sort the dictionary configuration. + # This is the only place where the order of the config should matter, when exporting in a specific style + def sort_dict(d: dict) -> dict: + """Takes a dict and sorts it, recursively.""" + sorted_dict = {x[0]: x[1] for x in sorted(d.items(), key=lambda e: e[0])} + + for k, v in d.items(): + if not isinstance(v, dict): + continue + sorted_dict[k] = sort_dict(v) + + return sorted_dict + + dump = sort_dict(dump) + + doc = atoml.document() + doc.add(atoml.comment("This is an autogenerated TOML document.")) + doc.add(atoml.comment("Directly run the config.py file to generate.")) + doc.add(atoml.nl()) + + doc.update(dump) + + # toml + with open(pathlib.Path(__file__).parent / "default_config.toml", "w") as f: + atoml.dump(doc, f) diff --git a/modmail/default_config.toml b/modmail/default_config.toml new file mode 100644 index 00000000..a4283f64 --- /dev/null +++ b/modmail/default_config.toml @@ -0,0 +1,18 @@ +# This is an autogenerated TOML document. +# Directly run the config.py file to generate. + +answer = 42 + +[bot] +prefix = "?" + +[colours] +base_embed_color = "7506394" + +[dev] +log_level = 20 + +[dev.mode] +develop = false +plugin_dev = false +production = true From 311ff5445c94fa145c0db18931194cd7ed36953f Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 20:44:10 -0400 Subject: [PATCH 08/99] fix: properly deserialize discord.Colour to a string representation Signed-off-by: onerandomusername --- modmail/config.py | 21 +++++++++++++-------- modmail/default_config.toml | 3 +-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 49e88dcc..91c27e82 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -38,10 +38,18 @@ def _generate_default_dict() -> defaultdict: class _ColourField(marshmallow.fields.Field): - """Class to convert a str or int into a color and deseriaze into a string.""" + """Class to convert a str or int into a color and deserialize into a string.""" class ColourConvert(discord.ext.commands.converter.ColourConverter): - def convert(self, argument: str) -> discord.Colour: + """Inherited discord.py colour converter.""" + + def convert(self, argument: typing.Union[str, int, discord.Colour]) -> discord.Colour: + """ + Convert an argument into a discord.Colour. + + This code was copied from discord.ext.commands.converter.ColourConverter. + Modified to not be async or need a context since it was not used in the first place. + """ if isinstance(argument, discord.Colour): return argument if not isinstance(argument, str): @@ -67,8 +75,8 @@ def convert(self, argument: str) -> discord.Colour: raise discord.ext.commands.converter.BadColourArgument(arg) return method() - def _serialize(self, value: typing.Any, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: - return str(value.value) + def _serialize(self, value: discord.Colour, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: + return "#" + hex(value.value)[2:].lower() def _deserialize( self, @@ -78,10 +86,7 @@ def _deserialize( **kwargs, ) -> str: if not isinstance(value, discord.Colour): - if isinstance(value, str): - value = self.ColourConvert().convert(value) - else: - value = discord.Colour(value) + value = self.ColourConvert().convert(value) return value diff --git a/modmail/default_config.toml b/modmail/default_config.toml index a4283f64..935be134 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,13 +1,12 @@ # This is an autogenerated TOML document. # Directly run the config.py file to generate. -answer = 42 [bot] prefix = "?" [colours] -base_embed_color = "7506394" +base_embed_color = "#7289da" [dev] log_level = 20 From 62aa7dcb39164652d39c8ace6850ed4a3a655815 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 20:48:29 -0400 Subject: [PATCH 09/99] feat: add configuration via a yaml cfg file Signed-off-by: onerandomusername --- .gitignore | 3 ++- modmail/config.py | 27 +++++++++++++++++++++------ modmail/default_config.yaml | 10 ++++++++++ poetry.lock | 9 ++++++--- pyproject.toml | 3 ++- 5 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 modmail/default_config.yaml diff --git a/.gitignore b/.gitignore index d737f24f..d52be37a 100644 --- a/.gitignore +++ b/.gitignore @@ -133,10 +133,11 @@ dmypy.json .idea/ # logs -logs +/logs/ # Configuration /modmail_config.toml +/modmail_config.yaml # Custom docker compose override docker-compose.override.yml diff --git a/modmail/config.py b/modmail/config.py index 91c27e82..98f14c7d 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,6 +1,5 @@ import inspect import logging -import os import pathlib import typing from collections import defaultdict @@ -16,12 +15,12 @@ import marshmallow.validate -_CWD = pathlib.Path(os.getcwd()) ENV_PREFIX = "MODMAIL_" -USER_CONFIG_TOML = _CWD / "modmail_config.toml" +USER_CONFIG_FILE_NAME = "modmail_config" +AUTO_GEN_FILE_NAME = "default_config" env = environs.Env(eager=False, expand_vars=True) -env.read_env(_CWD / ".env", recurse=False) +env.read_env(pathlib.Path.cwd() / ".env", recurse=False) def _generate_default_dict() -> defaultdict: @@ -31,11 +30,23 @@ def _generate_default_dict() -> defaultdict: unparsed_user_provided_cfg = defaultdict(lambda: marshmallow.missing) try: - with open(USER_CONFIG_TOML) as f: + with open(pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".toml")) as f: unparsed_user_provided_cfg.update(atoml.parse(f.read()).value) except FileNotFoundError: pass +# attempt to also check for a yaml file +try: + import yaml +except ImportError: + pass +else: + # check for an existing file + yaml_cfg = pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".yaml") + if yaml_cfg.is_file(): + with open(yaml_cfg, "r") as f: + unparsed_user_provided_cfg.update(yaml.load(f.read(), Loader=yaml.SafeLoader)) + class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deserialize into a string.""" @@ -270,5 +281,9 @@ def sort_dict(d: dict) -> dict: doc.update(dump) # toml - with open(pathlib.Path(__file__).parent / "default_config.toml", "w") as f: + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: atoml.dump(doc, f) + + # yaml + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: + yaml.dump(dump, f, indent=4) diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml new file mode 100644 index 00000000..c05fd99e --- /dev/null +++ b/modmail/default_config.yaml @@ -0,0 +1,10 @@ +bot: + prefix: '?' +colours: + base_embed_color: '#7289da' +dev: + log_level: 20 + mode: + develop: false + plugin_dev: false + production: true diff --git a/poetry.lock b/poetry.lock index 86aa99a8..07a497a0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1050,8 +1050,8 @@ cli = ["click (>=5.0)"] name = "pyyaml" version = "5.4.1" description = "YAML parser and emitter for Python" -category = "dev" -optional = false +category = "main" +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [[package]] @@ -1264,10 +1264,13 @@ python-versions = ">=3.6" docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +[extras] +yaml = ["PyYAML"] + [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "6cfec48a5437401e5e5aeece5efc90f48eae9e2903d6117c4f2cc850af33cdab" +content-hash = "74f9246891ee29ca657788bf12759a32664eb024b7e194ca214d19ed8c6784c5" [metadata.files] aiodns = [ diff --git a/pyproject.toml b/pyproject.toml index dad595b0..96395a3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,12 +25,13 @@ desert = "^2020.11.18" environs = "~=9.3.3" marshmallow = "~=3.13.0" python-dotenv = "^0.19.0" +PyYAML = { version = "^5.4.1", optional = true } typing-extensions = "^3.10.0.2" marshmallow-enum = "^1.5.1" [tool.poetry.extras] - +yaml = ["pyyaml"] [tool.poetry.dev-dependencies] # always needed From e4d9e1ed490f2cd740959f4f88203188072da6b7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 21:36:16 -0400 Subject: [PATCH 10/99] fix: allow log level 50, as documented Signed-off-by: onerandomusername --- modmail/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 98f14c7d..2acdaee6 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -177,8 +177,8 @@ class DevCfg: @log_level.validator def _log_level_validator(self, _: attr.Attribute, value: int) -> None: """Validate that log_level is within 0 to 50.""" - if value not in range(0, 50): - raise ValueError("log_level must be an integer within 0 to 50.") + if value not in range(0, 50 + 1): + raise ValueError("log_level must be an integer within 0 to 50, inclusive.") @attr.s(auto_attribs=True, slots=True) From 70128af93392ae316d160bc9cc2cd98031afaea7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 00:29:12 -0400 Subject: [PATCH 11/99] feat: made testable config system by refactor config file loading Signed-off-by: onerandomusername --- modmail/config.py | 276 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 228 insertions(+), 48 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 2acdaee6..6590f840 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,5 +1,6 @@ import inspect import logging +import os import pathlib import typing from collections import defaultdict @@ -15,12 +16,42 @@ import marshmallow.validate +try: + import yaml +except ImportError: + yaml = None + +__all__ = [ + "AUTO_GEN_FILE_NAME", + "DEFAULT_CONFIG_FILES", + "ENV_PREFIX", + "USER_CONFIG_FILE_NAME", + "CfgLoadError", + "Config", + "config", + "ConfigurationSchema", + "Bot", + "BotModeCfg", + "Cfg", + "Colours", + "DevCfg", + "convert_to_color", + "export_default_conf", + "get_config", + "get_default_config", + "load_toml", + "load_yaml", +] + +_CWD = pathlib.Path.cwd() + ENV_PREFIX = "MODMAIL_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" - -env = environs.Env(eager=False, expand_vars=True) -env.read_env(pathlib.Path.cwd() / ".env", recurse=False) +DEFAULT_CONFIG_FILES = [ + _CWD / (USER_CONFIG_FILE_NAME + ".yaml"), + _CWD / (USER_CONFIG_FILE_NAME + ".toml"), +] def _generate_default_dict() -> defaultdict: @@ -28,24 +59,10 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -unparsed_user_provided_cfg = defaultdict(lambda: marshmallow.missing) -try: - with open(pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".toml")) as f: - unparsed_user_provided_cfg.update(atoml.parse(f.read()).value) -except FileNotFoundError: - pass +class CfgLoadError(Exception): + """Exception if the configuration failed to load from a local file.""" -# attempt to also check for a yaml file -try: - import yaml -except ImportError: - pass -else: - # check for an existing file - yaml_cfg = pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".yaml") - if yaml_cfg.is_file(): - with open(yaml_cfg, "r") as f: - unparsed_user_provided_cfg.update(yaml.load(f.read(), Loader=yaml.SafeLoader)) + ... class _ColourField(marshmallow.fields.Field): @@ -195,8 +212,34 @@ class Cfg: dev: DevCfg = DevCfg() +# build configuration +ConfigurationSchema = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 + + +_CACHED_CONFIG: "Config" = None + + +@attr.s(auto_attribs=True, slots=True, kw_only=True) +class Config: + """ + Base configuration variable. Used across the entire bot for configuration variables. + + Holds two variables, default and user. + Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + + User is a Cfg schema instance, generated from a combination of the defaults, + user provided toml, and environment variables. + """ + + user: Cfg + schema: marshmallow.Schema + default: Cfg = Cfg() + + # find and build a bot class from our env -def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing.Dict = None) -> Bot: +def _build_bot_class( + klass: typing.Any, env: environs.Env, class_prefix: str = "", defaults: typing.Dict = None +) -> Bot: """ Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. @@ -223,40 +266,159 @@ def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing return klass(**kw) -_toml_bot_cfg = unparsed_user_provided_cfg["bot"] -unparsed_user_provided_cfg["bot"] = attr.asdict( - _build_bot_class(Bot, "BOT_", _toml_bot_cfg if _toml_bot_cfg is not marshmallow.missing else None) -) -del _toml_bot_cfg -# build configuration -Configuration = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 -USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.EXCLUDE) -DEFAULTS_CONFIG = Cfg() +def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: + """ + Load a configuration dictionary from the specified env file and environment variables. + All dependencies for this will always be installed. + """ + if env_file is None: + env_file = _CWD / ".env" + else: + env_file = pathlib.Path(env_file) -@attr.s(auto_attribs=True, slots=True, kw_only=True) -class Config: + env = environs.Env(eager=False, expand_vars=True) + env.read_env(".env", recurse=False) + + if not existing_cfg_dict: + existing_cfg_dict = defaultdict(_generate_default_dict) + + existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, "BOT_"))) + + return existing_cfg_dict + + +def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defaultdict: """ - Base configuration variable. Used across the entire bot for configuration variables. + Load a configuration dictionary from the specified toml file. - Holds two variables, default and user. - Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + All dependencies for this will always be installed. + """ + if path is None: + path = (_CWD / (USER_CONFIG_FILE_NAME + ".toml"),) + else: + # fully resolve path + path = pathlib.Path(path) - User is a Cfg schema instance, generated from a combination of the defaults, - user provided toml, and environment variables. + if not path.is_file(): + raise CfgLoadError("The provided toml file path is not a valid file.") + + try: + with open(path) as f: + loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) + if existing_cfg_dict is not None: + loaded_cfg.update(existing_cfg_dict) + return existing_cfg_dict + else: + return loaded_cfg + except Exception as e: + raise CfgLoadError from e + + +def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: """ + Load a configuration dictionary from the specified yaml file. - user: Cfg - default: Cfg - schema: marshmallow.Schema + The dependency for this may not be installed, as toml is already used elsewhere, so this is optional. + + In order to keep errors at a minimum, this function checks if both pyyaml is installed + and if a yaml configuration file exists before raising an exception. + """ + if path is None: + path = (_CWD / (USER_CONFIG_FILE_NAME + ".yaml"),) + else: + path = pathlib.Path(path) + + states = [ + ("The yaml library is not installed.", yaml is not None), + ("The provided yaml config path does not exist.", path.exists()), + ("The provided yaml config file is not a readable file.", path.is_file()), + ] + if errors := "\n".join(msg for msg, check in states if not check): + raise CfgLoadError(errors) + + try: + with open(path, "r") as f: + loaded_cfg = dict(yaml.load(f.read(), Loader=yaml.SafeLoader)) + if existing_cfg_dict is not None: + loaded_cfg.update(existing_cfg_dict) + return existing_cfg_dict + else: + return loaded_cfg + except Exception as e: + raise CfgLoadError from e + + +def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: bool = True) -> Config: + """ + Loads a configuration from the specified files. + Configuration will stop loading on the first existing file. + Default order checks yaml, then toml. -config = Config(user=USER_PROVIDED_CONFIG, default=DEFAULTS_CONFIG, schema=Configuration) + Supported file types are .toml or .yaml + """ + # load the env first + if load_env: + env_cfg = _load_env() + else: + env_cfg = None + + if files is None: + files = DEFAULT_CONFIG_FILES + elif len(files) == 0: + raise CfgLoadError("At least one file to load from must be provided.") + + loaded_config_dict: dict = None + for file in files: + if not isinstance(file, pathlib.Path): + file = pathlib.Path(file) + if not file.exists(): + # file does not exist + continue + + if file.suffix == ".toml" and atoml is not None: + loaded_config_dict = load_toml(file, existing_cfg_dict=env_cfg) + break + elif file.suffix == ".yaml" and yaml is not None: + loaded_config_dict = load_yaml(file, existing_cfg_dict=env_cfg) + break + else: + raise Exception( + "Provided configuration file is not of a supported type or " + "the required dependencies are not installed." + ) + + if loaded_config_dict is None: + raise CfgLoadError( + "Not gonna lie, this SHOULD be unreachable...\n" + "If you came across this as a consumer, please report this bug to our bug tracker." + ) + loaded_config_dict = ConfigurationSchema().load(loaded_config_dict) + return Config(user=loaded_config_dict, schema=ConfigurationSchema) + + +def get_config() -> Config: + """ + Helps to try to ensure that only one instance of the Config class exists. + + This means that all usage of the configuration is using the same configuration class. + """ + global _CACHED_CONFIG + if _CACHED_CONFIG is None: + _CACHED_CONFIG = _load_config() + return _CACHED_CONFIG -if __name__ == "__main__": - # if this file is run directly, export the configuration to a defaults file. - dump: dict = Configuration().dump(DEFAULTS_CONFIG) +def get_default_config() -> Cfg: + """Get the default configuration instance of the global Config instance.""" + return get_config().default + + +def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> None: + """Export default configuration to the preconfigured locations.""" + conf = get_default_config() + dump: dict = ConfigurationSchema().dump(conf) # Sort the dictionary configuration. # This is the only place where the order of the config should matter, when exporting in a specific style @@ -281,9 +443,27 @@ def sort_dict(d: dict) -> dict: doc.update(dump) # toml - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: - atoml.dump(doc, f) + if export_toml: + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: + atoml.dump(doc, f) # yaml - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: - yaml.dump(dump, f, indent=4) + if export_yaml is True or (yaml is not None and export_yaml is None): + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: + try: + yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + except AttributeError: + raise CfgLoadError( + "Tried to export the yaml configuration file but pyyaml is not installed." + ) from None + + +config = get_config() + + +if __name__ == "__main__": + # if this file is run directly, export the configuration to a defaults file. + import sys + + print("Exporting default configuration to desiginated default config files.") + sys.exit(export_default_conf()) From 4707d6ce3512a86f08b603c650d7340c2f27e95e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 00:50:26 -0400 Subject: [PATCH 12/99] tools: add pre-commit hook to automatically export default config if any modmail configuration file is edited, a pre-commit hook will run to check that the autogenerated files are up-to-date Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 10 ++++++++++ modmail/config.py | 10 +--------- scripts/__init__.py | 0 scripts/export_new_config_to_default_config.py | 16 ++++++++++++++++ 4 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/export_new_config_to_default_config.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e826392..cff28bbe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,16 @@ ## Pre-commit setup repos: + # its possible to put this at the bottom, but we want the pre-commit-hooks to check these files as well. + - repo: local + hooks: + - id: ensure-default-configuration-is-exported + name: Export default configuration + language: python + entry: poetry run python -m scripts.export_new_config_to_default_config + files: '(modmail\/(config.py|default_config(.toml|.yaml)))$' + additional_dependencies: [pyyaml==5.4.1] + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: diff --git a/modmail/config.py b/modmail/config.py index 6590f840..da169238 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -415,7 +415,7 @@ def get_default_config() -> Cfg: return get_config().default -def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> None: +def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> bool: """Export default configuration to the preconfigured locations.""" conf = get_default_config() dump: dict = ConfigurationSchema().dump(conf) @@ -459,11 +459,3 @@ def sort_dict(d: dict) -> dict: config = get_config() - - -if __name__ == "__main__": - # if this file is run directly, export the configuration to a defaults file. - import sys - - print("Exporting default configuration to desiginated default config files.") - sys.exit(export_default_conf()) diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py new file mode 100644 index 00000000..81102ccb --- /dev/null +++ b/scripts/export_new_config_to_default_config.py @@ -0,0 +1,16 @@ +""" +Exports the configuration to the configuration default files. + +This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. +""" +import sys + +import atoml # noqa: F401 +import yaml # noqa: F401 + +import modmail.config + + +if __name__ == "__main__": + print("Exporting configuration to default files. If they exist, overwriting their contents.") + sys.exit(modmail.config.export_default_conf(export_yaml=True)) From 26cc817f7f3998f5ba031b145a4abb6dcf43e5ee Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 01:47:31 -0400 Subject: [PATCH 13/99] tools: automatically export required environment varaibles to .env.template Signed-off-by: onerandomusername --- .env.template | 2 +- .pre-commit-config.yaml | 6 +- modmail/config.py | 49 +--------- modmail/default_config.toml | 2 +- modmail/default_config.yaml | 2 + .../export_new_config_to_default_config.py | 96 ++++++++++++++++++- 6 files changed, 105 insertions(+), 52 deletions(-) diff --git a/.env.template b/.env.template index 79023bad..66c582eb 100644 --- a/.env.template +++ b/.env.template @@ -1 +1 @@ -TOKEN="MyBotToken" +MODMAIL_BOT_TOKEN="MyBotToken" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cff28bbe..654e7dae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,8 +8,10 @@ repos: name: Export default configuration language: python entry: poetry run python -m scripts.export_new_config_to_default_config - files: '(modmail\/(config.py|default_config(.toml|.yaml)))$' - additional_dependencies: [pyyaml==5.4.1] + files: '(\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + additional_dependencies: + - atoml~=1.0.3 + - pyyaml~=5.4.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 diff --git a/modmail/config.py b/modmail/config.py index da169238..db95793b 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -25,6 +25,7 @@ "AUTO_GEN_FILE_NAME", "DEFAULT_CONFIG_FILES", "ENV_PREFIX", + "BOT_ENV_PREFIX", "USER_CONFIG_FILE_NAME", "CfgLoadError", "Config", @@ -36,7 +37,6 @@ "Colours", "DevCfg", "convert_to_color", - "export_default_conf", "get_config", "get_default_config", "load_toml", @@ -46,6 +46,7 @@ _CWD = pathlib.Path.cwd() ENV_PREFIX = "MODMAIL_" +BOT_ENV_PREFIX = "BOT_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" DEFAULT_CONFIG_FILES = [ @@ -140,6 +141,7 @@ class Bot: "required": True, "dump_only": True, "allow_none": False, + "modmail_export_filler": "MyBotToken", }, ) prefix: str = attr.ib( @@ -283,7 +285,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, "BOT_"))) + existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX))) return existing_cfg_dict @@ -415,47 +417,4 @@ def get_default_config() -> Cfg: return get_config().default -def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> bool: - """Export default configuration to the preconfigured locations.""" - conf = get_default_config() - dump: dict = ConfigurationSchema().dump(conf) - - # Sort the dictionary configuration. - # This is the only place where the order of the config should matter, when exporting in a specific style - def sort_dict(d: dict) -> dict: - """Takes a dict and sorts it, recursively.""" - sorted_dict = {x[0]: x[1] for x in sorted(d.items(), key=lambda e: e[0])} - - for k, v in d.items(): - if not isinstance(v, dict): - continue - sorted_dict[k] = sort_dict(v) - - return sorted_dict - - dump = sort_dict(dump) - - doc = atoml.document() - doc.add(atoml.comment("This is an autogenerated TOML document.")) - doc.add(atoml.comment("Directly run the config.py file to generate.")) - doc.add(atoml.nl()) - - doc.update(dump) - - # toml - if export_toml: - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: - atoml.dump(doc, f) - - # yaml - if export_yaml is True or (yaml is not None and export_yaml is None): - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: - try: - yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) - except AttributeError: - raise CfgLoadError( - "Tried to export the yaml configuration file but pyyaml is not installed." - ) from None - - config = get_config() diff --git a/modmail/default_config.toml b/modmail/default_config.toml index 935be134..de41e516 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,5 +1,5 @@ # This is an autogenerated TOML document. -# Directly run the config.py file to generate. +# Directly run scripts/export_new_config_to_default_config.py to generate. [bot] diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index c05fd99e..7ada283b 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -1,3 +1,5 @@ +# This is an autogenerated YAML document. +# Directly run scripts/export_new_config_to_default_config.py to generate. bot: prefix: '?' colours: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 81102ccb..26aba0ef 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -3,14 +3,104 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ +import pathlib import sys +from collections import defaultdict -import atoml # noqa: F401 -import yaml # noqa: F401 +import atoml +import attr +import marshmallow +import yaml import modmail.config +MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent +ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" + + +def export_default_conf() -> None: + """Export default configuration as both toml and yaml to the preconfigured locations.""" + conf = modmail.config.get_default_config() + dump: dict = modmail.config.ConfigurationSchema().dump(conf) + + # Sort the dictionary configuration. + # This is the only place where the order of the config should matter, when exporting in a specific style + def sort_dict(d: dict) -> dict: + """Takes a dict and sorts it, recursively.""" + sorted_dict = {x[0]: x[1] for x in sorted(d.items(), key=lambda e: e[0])} + + for k, v in d.items(): + if not isinstance(v, dict): + continue + sorted_dict[k] = sort_dict(v) + + return sorted_dict + + dump = sort_dict(dump) + autogen_gen_notice = f"Directly run scripts/{__file__.rsplit('/',1)[-1]!s} to generate." + doc = atoml.document() + doc.add(atoml.comment("This is an autogenerated TOML document.")) + doc.add(atoml.comment(autogen_gen_notice)) + doc.add(atoml.nl()) + + doc.update(dump) + + # toml + + with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml"), "w") as f: + atoml.dump(doc, f) + + # yaml + with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: + f.write("# This is an autogenerated YAML document.\n") + f.write(f"# {autogen_gen_notice}\n") + yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + + +def export_env_conf() -> None: + """ + Exports required configuration variables to .env.template. + + Does NOT export *all* settable variables! + + Export the *required* environment variables to `.env.template`. + Required environment variables are any Config.default.bot variables that default to marshmallow.missing + + Currently supported environment loading only loads to Config.user.bot, so we only need to worry + about those for the time being. Those values are additionally prefixed with `BOT_`. + + This means that in the end our exported variables are all prefixed with MODMAIL_BOT_, + and followed by the uppercase name of each field. + """ + env_prefix = modmail.config.ENV_PREFIX + modmail.config.BOT_ENV_PREFIX + default = modmail.config.get_default_config() + values = defaultdict(str) + fields = attr.fields(default.bot.__class__) + for attribute in fields: + if attribute.default is marshmallow.missing: + values[env_prefix + attribute.name.upper()] = attribute.metadata.get("modmail_export_filler", "") + + with open(ENV_EXPORT_FILE, "w") as f: + for k, v in values.items(): + f.write(k + '="' + v + '"\n') + + +def main() -> None: + """ + Exports the default configuration. + + There's two parts to this export. + First, export the default configuration to the default locations. + + Next, export the *required* configuration variables to the .env.template + + """ + export_default_conf() + + export_env_conf() + + if __name__ == "__main__": print("Exporting configuration to default files. If they exist, overwriting their contents.") - sys.exit(modmail.config.export_default_conf(export_yaml=True)) + sys.exit(main()) From b8d7080dfdc644b39d196cc19f6a7940619a2683 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 02:45:37 -0400 Subject: [PATCH 14/99] tools: add app.json to configuration exports Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 2 +- app.json | 2 +- modmail/config.py | 14 +++++-- modmail/utils/embeds.py | 2 +- modmail/utils/extensions.py | 2 +- .../export_new_config_to_default_config.py | 39 +++++++++++++++---- 6 files changed, 46 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 654e7dae..4f9f925a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: name: Export default configuration language: python entry: poetry run python -m scripts.export_new_config_to_default_config - files: '(\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' additional_dependencies: - atoml~=1.0.3 - pyyaml~=5.4.1 diff --git a/app.json b/app.json index 2a69903a..00c11d9a 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { "env": { "MODMAIL_BOT_TOKEN": { - "description": "Discord bot token. This is from https://discord.com/developers/applications", + "description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", "required": true } }, diff --git a/modmail/config.py b/modmail/config.py index db95793b..46eb2450 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -142,6 +142,7 @@ class Bot: "dump_only": True, "allow_none": False, "modmail_export_filler": "MyBotToken", + "modmail_env_description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", # noqa: E501 }, ) prefix: str = attr.ib( @@ -219,6 +220,7 @@ class Cfg: _CACHED_CONFIG: "Config" = None +_CACHED_DEFAULT: Cfg = None @attr.s(auto_attribs=True, slots=True, kw_only=True) @@ -396,8 +398,8 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: "Not gonna lie, this SHOULD be unreachable...\n" "If you came across this as a consumer, please report this bug to our bug tracker." ) - loaded_config_dict = ConfigurationSchema().load(loaded_config_dict) - return Config(user=loaded_config_dict, schema=ConfigurationSchema) + loaded_config_dict = ConfigurationSchema().load(loaded_config_dict, unknown=marshmallow.EXCLUDE) + return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) def get_config() -> Config: @@ -414,7 +416,11 @@ def get_config() -> Config: def get_default_config() -> Cfg: """Get the default configuration instance of the global Config instance.""" - return get_config().default + global _CACHED_DEFAULT + if _CACHED_DEFAULT is None: + _CACHED_DEFAULT = Cfg() + return _CACHED_DEFAULT -config = get_config() +config = get_config +default = get_default_config diff --git a/modmail/utils/embeds.py b/modmail/utils/embeds.py index 1e945c4c..9db685a8 100644 --- a/modmail/utils/embeds.py +++ b/modmail/utils/embeds.py @@ -6,7 +6,7 @@ from modmail.config import config -DEFAULT_COLOR = config.user.colours.base_embed_color +DEFAULT_COLOR = config().user.colours.base_embed_color original_init = discord.Embed.__init__ diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index 23f10880..b7bf7cae 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -35,7 +35,7 @@ def determine_bot_mode() -> int: """ bot_mode = 0 for mode in BotModes: - if getattr(config.user.dev.mode, unqualify(str(mode)).lower(), True): + if getattr(config().user.dev.mode, unqualify(str(mode)).lower(), True): bot_mode += mode.value return bot_mode diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 26aba0ef..34b22355 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -3,8 +3,10 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ +import json import pathlib import sys +import typing from collections import defaultdict import atoml @@ -17,6 +19,8 @@ MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" +APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" +METADATA_PREFIX = "modmail_" def export_default_conf() -> None: @@ -58,7 +62,7 @@ def sort_dict(d: dict) -> dict: yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) -def export_env_conf() -> None: +def export_env_and_app_json_conf() -> None: """ Exports required configuration variables to .env.template. @@ -75,30 +79,51 @@ def export_env_conf() -> None: """ env_prefix = modmail.config.ENV_PREFIX + modmail.config.BOT_ENV_PREFIX default = modmail.config.get_default_config() - values = defaultdict(str) + req_env_values: typing.Dict[str, attr.Attribute.metadata] = dict() fields = attr.fields(default.bot.__class__) for attribute in fields: if attribute.default is marshmallow.missing: - values[env_prefix + attribute.name.upper()] = attribute.metadata.get("modmail_export_filler", "") + req_env_values[env_prefix + attribute.name.upper()] = defaultdict(str, attribute.metadata) with open(ENV_EXPORT_FILE, "w") as f: - for k, v in values.items(): - f.write(k + '="' + v + '"\n') + for k, v in req_env_values.items(): + f.write(k + '="' + v[METADATA_PREFIX + "export_filler"] + '"\n') + + # the rest of this is designated for the app.json file + with open(APP_JSON_FILE) as f: + try: + app_json: typing.Dict = json.load(f) + except Exception as e: + print( + "Oops! Please ensure the app.json file is valid json! " + "If you've made manual edits, you may want to revert them." + ) + raise e + app_json_env = defaultdict(str) + for env_var, meta in req_env_values.items(): + app_json_env[env_var] = defaultdict( + str, {"description": meta[METADATA_PREFIX + "env_description"], "required": True} + ) + app_json["env"] = app_json_env + with open(APP_JSON_FILE, "w") as f: + json.dump(app_json, f, indent=4) + f.write("\n") def main() -> None: """ Exports the default configuration. - There's two parts to this export. + There's several parts to this export. First, export the default configuration to the default locations. Next, export the *required* configuration variables to the .env.template + In addition, export to app.json when exporting .env.template. """ export_default_conf() - export_env_conf() + export_env_and_app_json_conf() if __name__ == "__main__": From 19ddc22d7ba584c68ab10d9b630616099cce62b5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:00:47 -0400 Subject: [PATCH 15/99] minor: manaually update references to configuration files Signed-off-by: onerandomusername --- docs/contributing.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 3c0bad0b..4f351a13 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -152,14 +152,14 @@ $ poetry install ### Set up modmail config -1. Create a copy of `config-default.yml` named `config.yml` in the the `modmail/` directory. +1. Create a copy of `modmail/default_config.toml` named `modmail_config.toml` in the root of the repository. === "Linux, macOS"
```console - $ cp -v modmail/config-default.toml modmail/config.toml + $ cp -v modmail/default_config.toml modmail_config.toml ```
@@ -169,11 +169,15 @@ $ poetry install
```console - $ xcopy /f modmail/config-default.toml modmail/config.toml + $ xcopy /f modmail/default_config.toml modmail_config.toml ```
+!!! note + If you would like, there is optional support for yaml configuration. Make sure that pyyaml is installed with `poetry install --extras yaml`, + and copy modmail/defaults_config.yaml to modmail_config.yaml. + 2. Set the modmail bot prefix in `bot.prefix`. 3. In case you are a contributor set `dev.mode.plugin_dev` and `dev.mode.develop` to `true`. The `develop` variable enables the developer bot extensions and `plugin_dev` enables plugin-developer friendly bot extensions. 4. Create a text file named `.env` in your project root (that's the base folder of your repository): @@ -182,7 +186,7 @@ $ poetry install !!!note The entire file name is literally `.env` -5. Open the file with any text editor and write the bot token to the files in this format: `TOKEN="my_token"`. +5. Open the file with any text editor and write the bot token to the files in this format: `MODMAIL_BOT_TOKEN="my_token"`. ### Run The Project From 06aad80cabf128ead07bebd64b24df28c50001a3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:12:33 -0400 Subject: [PATCH 16/99] fix: add missing modmail dependencies to additional hook dependencies Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f9f925a..3c591291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,12 @@ repos: files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' additional_dependencies: - atoml~=1.0.3 + - attrs~=21.2.0 + - coloredlogs + - desert + - discord.py + - environs + - marshmallow~=3.13.0 - pyyaml~=5.4.1 - repo: https://github.com/pre-commit/pre-commit-hooks From cc502621f6f88fff50cfe3df6043c8e9a92d830e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:46:40 -0400 Subject: [PATCH 17/99] fix: no configuration files now loads the default config Signed-off-by: onerandomusername --- modmail/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 46eb2450..9596f351 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -394,10 +394,11 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: ) if loaded_config_dict is None: - raise CfgLoadError( - "Not gonna lie, this SHOULD be unreachable...\n" - "If you came across this as a consumer, please report this bug to our bug tracker." - ) + # load default configuration since no overrides were provided. + # that's actually done by default, so we replace the new dictionary + # with the one containing our required environment. + loaded_config_dict = env_cfg + loaded_config_dict = ConfigurationSchema().load(loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From bdc7d42a45399264d55fb3423b3e884573049972 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:47:21 -0400 Subject: [PATCH 18/99] fix: create a Config instance when adding it to the bot Signed-off-by: onerandomusername --- modmail/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/bot.py b/modmail/bot.py index 8d30426f..fb89711a 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -37,7 +37,7 @@ class ModmailBot(commands.Bot): logger: ModmailLogger = logging.getLogger(__name__) def __init__(self, **kwargs): - self.config = config + self.config = config() self.start_time: t.Optional[arrow.Arrow] = None # arrow.utcnow() self.http_session: t.Optional[ClientSession] = None From 29de36ee11d51a9232dc7a682f63526facda686c Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 15:15:11 -0400 Subject: [PATCH 19/99] fix: load prefix from modmail_config.toml if not in .env Signed-off-by: onerandomusername --- .env.template | 1 + .pre-commit-config.yaml | 1 + app.json | 4 ++++ modmail/config.py | 47 ++++++++++++++++++++++++----------------- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/.env.template b/.env.template index 66c582eb..521ba21f 100644 --- a/.env.template +++ b/.env.template @@ -1 +1,2 @@ MODMAIL_BOT_TOKEN="MyBotToken" +MODMAIL_BOT_PREFIX="" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c591291..f06fe14f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,7 @@ repos: - id: check-yaml exclude: 'mkdocs.yml' # Exclude all mkdocs.yml as they use tags i.e. `!!!` - id: pretty-format-json + exclude: 'app.json' args: [--indent=4, --autofix] - id: end-of-file-fixer - id: no-commit-to-branch diff --git a/app.json b/app.json index 00c11d9a..d48489cc 100644 --- a/app.json +++ b/app.json @@ -3,6 +3,10 @@ "MODMAIL_BOT_TOKEN": { "description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", "required": true + }, + "MODMAIL_BOT_PREFIX": { + "description": "", + "required": true } }, "name": "Modmail Bot", diff --git a/modmail/config.py b/modmail/config.py index 9596f351..dd714c27 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -60,6 +60,21 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) +def _recursive_dict_update(d1: dict, d2: dict) -> defaultdict: + """ + Recursively update a dictionary with the values from another dictionary. + + Serves to ensure that all keys from both exist. + """ + d1.update(d2) + for k, v in d1.items(): + if isinstance(v, dict) and isinstance(d2.get(k, None), dict): + d1[k] = _recursive_dict_update(v, d2[k]) + elif v is marshmallow.missing and d2.get(k, marshmallow.missing) is not marshmallow.missing: + d1[k] = d2[k] + return defaultdict(lambda: marshmallow.missing, d1) + + class CfgLoadError(Exception): """Exception if the configuration failed to load from a local file.""" @@ -146,11 +161,8 @@ class Bot: }, ) prefix: str = attr.ib( - default="?", - converter=lambda x: "?" if x is None else x, - metadata={ - "allow_none": False, - }, + default=marshmallow.missing, + converter=lambda x: "?" if x is marshmallow.missing else x, ) @@ -266,6 +278,8 @@ def _build_bot_class( kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) if defaults and kw[var.name] is None: kw[var.name] = defaults[var.name] + elif kw[var.name] is None: + kw[var.name] = marshmallow.missing return klass(**kw) @@ -287,7 +301,9 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX))) + existing_cfg_dict["bot"] = _recursive_dict_update( + attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)), existing_cfg_dict["bot"] + ) return existing_cfg_dict @@ -311,8 +327,8 @@ def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defau with open(path) as f: loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) if existing_cfg_dict is not None: - loaded_cfg.update(existing_cfg_dict) - return existing_cfg_dict + _recursive_dict_update(loaded_cfg, existing_cfg_dict) + return loaded_cfg else: return loaded_cfg except Exception as e: @@ -362,11 +378,7 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: Supported file types are .toml or .yaml """ - # load the env first - if load_env: - env_cfg = _load_env() - else: - env_cfg = None + env_cfg = None if files is None: files = DEFAULT_CONFIG_FILES @@ -393,13 +405,10 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: "the required dependencies are not installed." ) - if loaded_config_dict is None: - # load default configuration since no overrides were provided. - # that's actually done by default, so we replace the new dictionary - # with the one containing our required environment. - loaded_config_dict = env_cfg + if load_env: + loaded_config_dict = _load_env(existing_cfg_dict=loaded_config_dict) - loaded_config_dict = ConfigurationSchema().load(loaded_config_dict, unknown=marshmallow.EXCLUDE) + loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From 2166f8c7844f939bd0cf1d54d4615e8956cfc262 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 16:54:11 -0400 Subject: [PATCH 20/99] fix: ignore extra configuration variables Signed-off-by: onerandomusername --- modmail/config.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/modmail/config.py b/modmail/config.py index dd714c27..e45d86e2 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -302,7 +302,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d existing_cfg_dict = defaultdict(_generate_default_dict) existing_cfg_dict["bot"] = _recursive_dict_update( - attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)), existing_cfg_dict["bot"] + existing_cfg_dict["bot"], attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)) ) return existing_cfg_dict @@ -369,6 +369,29 @@ def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: raise CfgLoadError from e +DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) + + +def _remove_extra_values(klass: type, dit: DictT) -> DictT: + """ + Remove extra values from the provided dict which don't fit into the provided klass recursively. + + klass must be an attr.s class. + """ + fields = attr.fields_dict(klass) + cleared_dict = dit.copy() + for k in dit: + if k not in fields: + del cleared_dict[k] + elif isinstance(cleared_dict[k], dict): + if attr.has((new_klass := fields.get(k, None)).type): + cleared_dict[k] = _remove_extra_values(new_klass.type, cleared_dict[k]) + else: + # delete this dict + del cleared_dict[k] + return cleared_dict + + def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: bool = True) -> Config: """ Loads a configuration from the specified files. @@ -408,6 +431,12 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: if load_env: loaded_config_dict = _load_env(existing_cfg_dict=loaded_config_dict) + # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them + # CONTRARY to the marshmallow.EXCLUDE below. + # They will cause errors. + # Extra configuration values are okay, we aren't trying to be strict here. + loaded_config_dict = _remove_extra_values(Cfg, loaded_config_dict) + loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From be6b698393d5cfc5c8fcb70e9b8e206d7f54b7d9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 19:13:30 -0400 Subject: [PATCH 21/99] minor: use attr helper method instead of reimplementing it checked with a debug command while in a debug session '{x for x in attr.fields(klass)} == attribs' is True. They are exactly the same, so I've switched to using the attr method instead of rolling my own. Signed-off-by: onerandomusername --- modmail/config.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index e45d86e2..2c97014a 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -266,15 +266,10 @@ def _build_bot_class( defaults = defaultdict(lambda: marshmallow.missing) else: defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) - attribs: typing.Set[attr.Attribute] = set() - for a in dir(klass.__attrs_attrs__): - if hasattr(klass.__attrs_attrs__, a): - if isinstance(attribute := getattr(klass.__attrs_attrs__, a), attr.Attribute): - attribs.add(attribute) - # read the env vars from the above + with env.prefixed(ENV_PREFIX): kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing - for var in attribs: + for var in attr.fields(klass): kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) if defaults and kw[var.name] is None: kw[var.name] = defaults[var.name] From 4a367c4c42329af8a4a85947246038886300b9e1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 19:14:42 -0400 Subject: [PATCH 22/99] chore: switch _load_config to pack positional args rather than require an iterable Signed-off-by: onerandomusername --- modmail/config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 2c97014a..97143ed7 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -387,7 +387,7 @@ def _remove_extra_values(klass: type, dit: DictT) -> DictT: return cleared_dict -def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: bool = True) -> Config: +def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: """ Loads a configuration from the specified files. @@ -398,10 +398,8 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: """ env_cfg = None - if files is None: + if len(files) == 0: files = DEFAULT_CONFIG_FILES - elif len(files) == 0: - raise CfgLoadError("At least one file to load from must be provided.") loaded_config_dict: dict = None for file in files: From 4e085d72e32db2f41828a7b944c5c59dea54fab9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 20:13:41 -0400 Subject: [PATCH 23/99] fix: rewrite environment variable parser to support all configuration options Signed-off-by: onerandomusername --- modmail/config.py | 64 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 97143ed7..7af74526 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -10,7 +10,7 @@ import desert import discord import discord.ext.commands.converter -import environs +import dotenv import marshmallow import marshmallow.fields import marshmallow.validate @@ -54,6 +54,11 @@ _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] +# load env before we do *anything* +# TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. +# TODO: add the above feature to the other configuration locations too. +dotenv.load_dotenv(_CWD / ".env") + def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" @@ -252,29 +257,57 @@ class Config: default: Cfg = Cfg() +ClassT = typing.TypeVar("ClassT", bound=type) + + # find and build a bot class from our env -def _build_bot_class( - klass: typing.Any, env: environs.Env, class_prefix: str = "", defaults: typing.Dict = None -) -> Bot: +def _build_class( + klass: ClassT, + env: typing.Dict[str, str] = None, + env_prefix: str = None, + *, + dotenv_file: os.PathLike = None, + defaults: typing.Dict = None, +) -> ClassT: """ Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. + Defaults to getting the environment variables with dotenv. + Also can parse from a provided dictionary of environment variables. If defaults is provided, uses a value from there if the environment variable is not set or is None. """ + if env_prefix is None: + env_prefix = ENV_PREFIX + + if env is None: + if dotenv_file is not None: + dotenv.load_dotenv(dotenv_file) + env = os.environ.copy() + # get the attributes of the provided class if defaults is None: defaults = defaultdict(lambda: marshmallow.missing) else: defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) - with env.prefixed(ENV_PREFIX): - kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing - for var in attr.fields(klass): - kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) - if defaults and kw[var.name] is None: - kw[var.name] = defaults[var.name] - elif kw[var.name] is None: - kw[var.name] = marshmallow.missing + kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing + + for var in attr.fields(klass): + if attr.has(var.type): + # var is an attrs class too + kw[var.name] = _build_class( + var.type, + env=env, + env_prefix=env_prefix + var.name.upper() + "_", + defaults=defaults.get(var.name, None), + ) + else: + kw[var.name] = env.get(env_prefix + var.name.upper(), None) + if kw[var.name] is None: + if defaults: + kw[var.name] = defaults[var.name] + else: + del kw[var.name] return klass(**kw) @@ -290,14 +323,11 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d else: env_file = pathlib.Path(env_file) - env = environs.Env(eager=False, expand_vars=True) - env.read_env(".env", recurse=False) - if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict["bot"] = _recursive_dict_update( - existing_cfg_dict["bot"], attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)) + existing_cfg_dict = _recursive_dict_update( + existing_cfg_dict, attr.asdict(_build_class(Cfg, dotenv_file=env_file)) ) return existing_cfg_dict From 449593838a5896fdce782642b5d5c267c8bcf42b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 21:26:30 -0400 Subject: [PATCH 24/99] feat: add ConfigMetadata part 1 Signed-off-by: onerandomusername --- .env.template | 3 +- .pre-commit-config.yaml | 1 + app.json | 6 +-- modmail/config.py | 52 +++++++++++++++++-- .../export_new_config_to_default_config.py | 16 ++++-- 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/.env.template b/.env.template index 521ba21f..d8615894 100644 --- a/.env.template +++ b/.env.template @@ -1,2 +1 @@ -MODMAIL_BOT_TOKEN="MyBotToken" -MODMAIL_BOT_PREFIX="" +MODMAIL_BOT_TOKEN='Bot Token' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f06fe14f..b6136883 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: - discord.py - environs - marshmallow~=3.13.0 + - python-dotenv - pyyaml~=5.4.1 - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/app.json b/app.json index d48489cc..b3b04bc4 100644 --- a/app.json +++ b/app.json @@ -1,11 +1,7 @@ { "env": { "MODMAIL_BOT_TOKEN": { - "description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", - "required": true - }, - "MODMAIL_BOT_PREFIX": { - "description": "", + "description": "Discord bot token. Required to log in to discord.", "required": true } }, diff --git a/modmail/config.py b/modmail/config.py index 7af74526..26d7aa23 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -44,7 +44,7 @@ ] _CWD = pathlib.Path.cwd() - +METADATA_TABLE = "modmail_metadata" ENV_PREFIX = "MODMAIL_" BOT_ENV_PREFIX = "BOT_" USER_CONFIG_FILE_NAME = "modmail_config" @@ -144,6 +144,38 @@ def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Col return _ColourField.ColourConvert().convert(col) +@attr.dataclass(frozen=True, kw_only=True) +class ConfigMetadata: + """ + Cfg metadata. This is intended to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. + + Nearly all of these values are optional, save for the description. + All of them are keyword only. + In addition, all instances of this class are frozen; they are not meant to be changed after creation. + + These values are meant to be used for the configuration UI and exports. + In relation to that, most are optional. However, *all* instances must have a description. + """ + + # what to refer to for the end user + description: str = attr.ib() + canconical_name: str = None + # for those configuration options where the description just won't cut it. + extended_description: str = None + # for the variables to export to the environment, what value should be prefilled + export_environment_prefill: str = None + # comment provided above or beside the default configuration option in exports + export_note: str = None + # this only has an effect if required is not True + # required variables will always be exported, but if this is a commonly changed var, this should be True. + export_to_env_template: bool = False + + @description.validator + def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: + if not isinstance(value, attrib.type): + raise ValueError(f"description must be of {attrib.type}") from None + + @attr.s(auto_attribs=True, slots=True) class Bot: """ @@ -161,12 +193,24 @@ class Bot: "required": True, "dump_only": True, "allow_none": False, - "modmail_export_filler": "MyBotToken", - "modmail_env_description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", # noqa: E501 + METADATA_TABLE: ConfigMetadata( + canconical_name="Bot Token", + description="Discord bot token. Required to log in to discord.", + export_environment_prefill="Bot Token", + extended_description="This is obtainable from https://discord.com/developers/applications", + export_to_env_template=True, + ), }, ) prefix: str = attr.ib( - default=marshmallow.missing, + default="?", + metadata={ + METADATA_TABLE: ConfigMetadata( + canconical_name="Command Prefix", + description="Command prefix.", + export_to_env_template=True, + ) + }, converter=lambda x: "?" if x is marshmallow.missing else x, ) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 34b22355..fc6abedb 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -11,6 +11,7 @@ import atoml import attr +import dotenv import marshmallow import yaml @@ -85,9 +86,12 @@ def export_env_and_app_json_conf() -> None: if attribute.default is marshmallow.missing: req_env_values[env_prefix + attribute.name.upper()] = defaultdict(str, attribute.metadata) - with open(ENV_EXPORT_FILE, "w") as f: - for k, v in req_env_values.items(): - f.write(k + '="' + v[METADATA_PREFIX + "export_filler"] + '"\n') + # dotenv modifies currently existing files, but we want to erase the current file + ENV_EXPORT_FILE.unlink(missing_ok=True) + ENV_EXPORT_FILE.touch() + + for k, v in req_env_values.items(): + dotenv.set_key(ENV_EXPORT_FILE, k, v[modmail.config.METADATA_TABLE].export_environment_prefill) # the rest of this is designated for the app.json file with open(APP_JSON_FILE) as f: @@ -102,7 +106,11 @@ def export_env_and_app_json_conf() -> None: app_json_env = defaultdict(str) for env_var, meta in req_env_values.items(): app_json_env[env_var] = defaultdict( - str, {"description": meta[METADATA_PREFIX + "env_description"], "required": True} + str, + { + "description": meta[modmail.config.METADATA_TABLE].description, + "required": meta.get("required", True), + }, ) app_json["env"] = app_json_env with open(APP_JSON_FILE, "w") as f: From 0085f2aee87e94edd1b2e11c9a5f2db297d1b0b5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 21:39:28 -0400 Subject: [PATCH 25/99] fix recursive dict update overriding set values Signed-off-by: onerandomusername --- modmail/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 26d7aa23..5c002797 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -71,12 +71,17 @@ def _recursive_dict_update(d1: dict, d2: dict) -> defaultdict: Serves to ensure that all keys from both exist. """ - d1.update(d2) for k, v in d1.items(): if isinstance(v, dict) and isinstance(d2.get(k, None), dict): d1[k] = _recursive_dict_update(v, d2[k]) - elif v is marshmallow.missing and d2.get(k, marshmallow.missing) is not marshmallow.missing: + elif (v is marshmallow.missing or v is None) and d2.get( + k, marshmallow.missing + ) is not marshmallow.missing: d1[k] = d2[k] + for k, v in d2.items(): + no = d1.get(k, None) + if no is marshmallow.missing or no is None: + d1[k] = v return defaultdict(lambda: marshmallow.missing, d1) From 9417d3c199c7dce601d82b8d88418a9d5ab05502 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 21:45:48 -0400 Subject: [PATCH 26/99] chore: update export docstring Signed-off-by: onerandomusername --- scripts/export_new_config_to_default_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index fc6abedb..10d9bbb1 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -72,8 +72,9 @@ def export_env_and_app_json_conf() -> None: Export the *required* environment variables to `.env.template`. Required environment variables are any Config.default.bot variables that default to marshmallow.missing - Currently supported environment loading only loads to Config.user.bot, so we only need to worry - about those for the time being. Those values are additionally prefixed with `BOT_`. + TODO: as of right now, all configuration values can be configured with environment variables. + However, this method only exports the MODMAIL_BOT_ *required* varaibles to the template files. + This will be rewritten to support full unload and the new modmail.config.ConfigMetadata class. This means that in the end our exported variables are all prefixed with MODMAIL_BOT_, and followed by the uppercase name of each field. From bb1d956eee649d1dd071ad52d7242cb9ec874405 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 22:48:45 -0400 Subject: [PATCH 27/99] fix: respect priorty of configuration sources Signed-off-by: onerandomusername --- .env.template | 1 + app.json | 5 +++ modmail/config.py | 37 ++++++++++++------- .../export_new_config_to_default_config.py | 8 +++- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.env.template b/.env.template index d8615894..03a02729 100644 --- a/.env.template +++ b/.env.template @@ -1 +1,2 @@ MODMAIL_BOT_TOKEN='Bot Token' +MODMAIL_BOT_PREFIX='?' diff --git a/app.json b/app.json index b3b04bc4..e0d64443 100644 --- a/app.json +++ b/app.json @@ -3,6 +3,11 @@ "MODMAIL_BOT_TOKEN": { "description": "Discord bot token. Required to log in to discord.", "required": true + }, + "MODMAIL_BOT_PREFIX": { + "description": "Command prefix.", + "required": false, + "value": "?" } }, "name": "Modmail Bot", diff --git a/modmail/config.py b/modmail/config.py index 5c002797..d25d7986 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -65,7 +65,7 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -def _recursive_dict_update(d1: dict, d2: dict) -> defaultdict: +def _recursive_dict_update(d1: dict, d2: dict, attr_cls: type = None) -> defaultdict: """ Recursively update a dictionary with the values from another dictionary. @@ -175,6 +175,10 @@ class ConfigMetadata: # required variables will always be exported, but if this is a commonly changed var, this should be True. export_to_env_template: bool = False + # app json is slightly different, so there's additional options for them + app_json_default: str = None + app_json_required: bool = None + @description.validator def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: if not isinstance(value, attrib.type): @@ -208,11 +212,14 @@ class Bot: }, ) prefix: str = attr.ib( - default="?", + default=marshmallow.missing, metadata={ METADATA_TABLE: ConfigMetadata( canconical_name="Command Prefix", description="Command prefix.", + export_environment_prefill="?", + app_json_default="?", + app_json_required=False, export_to_env_template=True, ) }, @@ -261,7 +268,7 @@ class DevCfg: log_level: int = attr.ib(default=logging.INFO) @log_level.validator - def _log_level_validator(self, _: attr.Attribute, value: int) -> None: + def _log_level_validator(self, a: attr.Attribute, value: int) -> None: """Validate that log_level is within 0 to 50.""" if value not in range(0, 50 + 1): raise ValueError("log_level must be an integer within 0 to 50, inclusive.") @@ -335,11 +342,11 @@ def _build_class( # get the attributes of the provided class if defaults is None: - defaults = defaultdict(lambda: marshmallow.missing) + defaults = defaultdict(lambda: None) else: - defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) + defaults = defaultdict(lambda: None, defaults.copy()) - kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing + kw = defaultdict(lambda: None) # any missing required vars provide as missing for var in attr.fields(klass): if attr.has(var.type): @@ -353,8 +360,12 @@ def _build_class( else: kw[var.name] = env.get(env_prefix + var.name.upper(), None) if kw[var.name] is None: - if defaults: + if defaults is not None and ( + (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing + ): kw[var.name] = defaults[var.name] + elif var.default is not attr.NOTHING: # check for var default + kw[var.name] = var.default else: del kw[var.name] @@ -375,9 +386,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict = _recursive_dict_update( - existing_cfg_dict, attr.asdict(_build_class(Cfg, dotenv_file=env_file)) - ) + existing_cfg_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) return existing_cfg_dict @@ -401,7 +410,7 @@ def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defau with open(path) as f: loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) if existing_cfg_dict is not None: - _recursive_dict_update(loaded_cfg, existing_cfg_dict) + loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) return loaded_cfg else: return loaded_cfg @@ -433,10 +442,10 @@ def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: try: with open(path, "r") as f: - loaded_cfg = dict(yaml.load(f.read(), Loader=yaml.SafeLoader)) + loaded_cfg = defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) if existing_cfg_dict is not None: - loaded_cfg.update(existing_cfg_dict) - return existing_cfg_dict + loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) + return loaded_cfg else: return loaded_cfg except Exception as e: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 10d9bbb1..584563db 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -106,13 +106,17 @@ def export_env_and_app_json_conf() -> None: raise e app_json_env = defaultdict(str) for env_var, meta in req_env_values.items(): - app_json_env[env_var] = defaultdict( + options = defaultdict( str, { "description": meta[modmail.config.METADATA_TABLE].description, - "required": meta.get("required", True), + "required": meta[modmail.config.METADATA_TABLE].app_json_required + or meta.get("required", False), }, ) + if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: + options["value"] = value + app_json_env[env_var] = options app_json["env"] = app_json_env with open(APP_JSON_FILE, "w") as f: json.dump(app_json, f, indent=4) From d5f1290960ac33e9093bca6cda96e56ce13674f9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 23:00:17 -0400 Subject: [PATCH 28/99] fix pre-commit maybe Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6136883..ed670ebc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: language: python entry: poetry run python -m scripts.export_new_config_to_default_config files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + require_serial: true additional_dependencies: - atoml~=1.0.3 - attrs~=21.2.0 From f3c3326b8260cea57cd12d405a28d98e63d59c56 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 03:41:52 -0400 Subject: [PATCH 29/99] nit: rename config class Bot to BotCfg to prevent future confusion Signed-off-by: onerandomusername --- modmail/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index d25d7986..5f4fddfb 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -31,7 +31,7 @@ "Config", "config", "ConfigurationSchema", - "Bot", + "BotCfg", "BotModeCfg", "Cfg", "Colours", @@ -186,7 +186,7 @@ def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> No @attr.s(auto_attribs=True, slots=True) -class Bot: +class BotCfg: """ Values that are configuration for the bot itself. @@ -283,7 +283,7 @@ class Cfg: we can get a clean default variable if we don't pass anything. """ - bot: Bot = Bot() + bot: BotCfg = BotCfg() colours: Colours = Colours() dev: DevCfg = DevCfg() From 6e14be0b1740dcd33c28a0322476cf93d9ef772c Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 04:14:13 -0400 Subject: [PATCH 30/99] fix: add ConfigMetadata part 2 Signed-off-by: onerandomusername --- modmail/config.py | 64 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 5f4fddfb..e8e33578 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -149,7 +149,7 @@ def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Col return _ColourField.ColourConvert().convert(col) -@attr.dataclass(frozen=True, kw_only=True) +@attr.frozen(kw_only=True) class ConfigMetadata: """ Cfg metadata. This is intended to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. @@ -179,6 +179,10 @@ class ConfigMetadata: app_json_default: str = None app_json_required: bool = None + # hidden, eg log_level + # hidden values mean they do not show up in the bot configuration menu + hidden: bool = False + @description.validator def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: if not isinstance(value, attrib.type): @@ -241,10 +245,32 @@ class BotModeCfg: marshmallow.fields.Constant(True), default=True, converter=lambda _: True, - metadata={"dump_default": True, "dump_only": True}, + metadata={ + "dump_default": True, + "dump_only": True, + METADATA_TABLE: ConfigMetadata( + description="Production Mode. This is not changeable.", + ), + }, + ) + develop: bool = attr.ib( + default=False, + metadata={ + "allow_none": False, + METADATA_TABLE: ConfigMetadata( + description="Bot developer mode. Enables additional developer specific features.", + ), + }, + ) + plugin_dev: bool = attr.ib( + default=False, + metadata={ + "allow_none": False, + METADATA_TABLE: ConfigMetadata( + description="Plugin developer mode. Enables additional plugin developer specific features.", + ), + }, ) - develop: bool = attr.ib(default=False, metadata={"allow_none": False}) - plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) @attr.s(auto_attribs=True, slots=True) @@ -256,7 +282,14 @@ class Colours: """ base_embed_color: discord.Colour = desert.ib( - _ColourField(), default="0x7289DA", converter=convert_to_color + _ColourField(), + default="0x7289DA", + converter=convert_to_color, + metadata={ + METADATA_TABLE: ConfigMetadata( + description="Default embed colour for all embeds without a designated colour.", + ) + }, ) @@ -265,7 +298,15 @@ class DevCfg: """Developer configuration. These values should not be changed unless you know what you're doing.""" mode: BotModeCfg = BotModeCfg() - log_level: int = attr.ib(default=logging.INFO) + log_level: int = attr.ib( + default=logging.INFO, + metadata={ + METADATA_TABLE: ConfigMetadata( + description="Logging level.", + hidden=True, + ) + }, + ) @log_level.validator def _log_level_validator(self, a: attr.Attribute, value: int) -> None: @@ -285,7 +326,16 @@ class Cfg: bot: BotCfg = BotCfg() colours: Colours = Colours() - dev: DevCfg = DevCfg() + dev: DevCfg = attr.ib( + default=DevCfg(), + metadata={ + METADATA_TABLE: ConfigMetadata( + description="Developer configuration. " + "Only change these values if you know what you're doing.", + hidden=True, + ) + }, + ) # build configuration From c05e0bb3715e7d321e059d2a4f7e2aad942a8256 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 04:34:41 -0400 Subject: [PATCH 31/99] chore: remove unused dict manipulation method removed the now defunct _recursive_dict_update method Signed-off-by: onerandomusername --- modmail/config.py | 44 ++++++-------------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index e8e33578..3d4cd900 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -65,26 +65,6 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -def _recursive_dict_update(d1: dict, d2: dict, attr_cls: type = None) -> defaultdict: - """ - Recursively update a dictionary with the values from another dictionary. - - Serves to ensure that all keys from both exist. - """ - for k, v in d1.items(): - if isinstance(v, dict) and isinstance(d2.get(k, None), dict): - d1[k] = _recursive_dict_update(v, d2[k]) - elif (v is marshmallow.missing or v is None) and d2.get( - k, marshmallow.missing - ) is not marshmallow.missing: - d1[k] = d2[k] - for k, v in d2.items(): - no = d1.get(k, None) - if no is marshmallow.missing or no is None: - d1[k] = v - return defaultdict(lambda: marshmallow.missing, d1) - - class CfgLoadError(Exception): """Exception if the configuration failed to load from a local file.""" @@ -441,7 +421,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d return existing_cfg_dict -def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defaultdict: +def load_toml(path: os.PathLike = None) -> defaultdict: """ Load a configuration dictionary from the specified toml file. @@ -458,17 +438,12 @@ def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defau try: with open(path) as f: - loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) - if existing_cfg_dict is not None: - loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) - return loaded_cfg - else: - return loaded_cfg + return defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) except Exception as e: raise CfgLoadError from e -def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: +def load_yaml(path: os.PathLike) -> dict: """ Load a configuration dictionary from the specified yaml file. @@ -492,12 +467,7 @@ def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: try: with open(path, "r") as f: - loaded_cfg = defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) - if existing_cfg_dict is not None: - loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) - return loaded_cfg - else: - return loaded_cfg + return defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) except Exception as e: raise CfgLoadError from e @@ -534,8 +504,6 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: Supported file types are .toml or .yaml """ - env_cfg = None - if len(files) == 0: files = DEFAULT_CONFIG_FILES @@ -548,10 +516,10 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: continue if file.suffix == ".toml" and atoml is not None: - loaded_config_dict = load_toml(file, existing_cfg_dict=env_cfg) + loaded_config_dict = load_toml(file) break elif file.suffix == ".yaml" and yaml is not None: - loaded_config_dict = load_yaml(file, existing_cfg_dict=env_cfg) + loaded_config_dict = load_yaml(file) break else: raise Exception( From 444aca17898952e69557ac8b68a16d887ba627dd Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 18:12:13 -0400 Subject: [PATCH 32/99] chore: implement new metadata options on cfg default export Signed-off-by: onerandomusername --- modmail/config.py | 9 +- .../export_new_config_to_default_config.py | 102 ++++++++++++------ 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 3d4cd900..4e6ae89d 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -23,10 +23,9 @@ __all__ = [ "AUTO_GEN_FILE_NAME", - "DEFAULT_CONFIG_FILES", "ENV_PREFIX", - "BOT_ENV_PREFIX", "USER_CONFIG_FILE_NAME", + "USER_CONFIG_FILES", "CfgLoadError", "Config", "config", @@ -46,10 +45,9 @@ _CWD = pathlib.Path.cwd() METADATA_TABLE = "modmail_metadata" ENV_PREFIX = "MODMAIL_" -BOT_ENV_PREFIX = "BOT_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" -DEFAULT_CONFIG_FILES = [ +USER_CONFIG_FILES = [ _CWD / (USER_CONFIG_FILE_NAME + ".yaml"), _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] @@ -156,6 +154,7 @@ class ConfigMetadata: export_to_env_template: bool = False # app json is slightly different, so there's additional options for them + export_to_app_json: bool = None app_json_default: str = None app_json_required: bool = None @@ -505,7 +504,7 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: Supported file types are .toml or .yaml """ if len(files) == 0: - files = DEFAULT_CONFIG_FILES + files = USER_CONFIG_FILES loaded_config_dict: dict = None for file in files: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 584563db..d3098400 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -12,7 +12,6 @@ import atoml import attr import dotenv -import marshmallow import yaml import modmail.config @@ -21,13 +20,21 @@ MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" -METADATA_PREFIX = "modmail_" + +METADATA_TABLE = modmail.config.METADATA_TABLE + + +class MetadataDict(typing.TypedDict): + """Typed metadata. This has a possible risk given that the modmail_metadata variable is defined.""" + + modmail_metadata: modmail.config.ConfigMetadata + required: bool def export_default_conf() -> None: """Export default configuration as both toml and yaml to the preconfigured locations.""" - conf = modmail.config.get_default_config() - dump: dict = modmail.config.ConfigurationSchema().dump(conf) + default = modmail.config.get_default_config() + dump: dict = modmail.config.ConfigurationSchema().dump(default) # Sort the dictionary configuration. # This is the only place where the order of the config should matter, when exporting in a specific style @@ -70,31 +77,42 @@ def export_env_and_app_json_conf() -> None: Does NOT export *all* settable variables! Export the *required* environment variables to `.env.template`. - Required environment variables are any Config.default.bot variables that default to marshmallow.missing - - TODO: as of right now, all configuration values can be configured with environment variables. - However, this method only exports the MODMAIL_BOT_ *required* varaibles to the template files. - This will be rewritten to support full unload and the new modmail.config.ConfigMetadata class. - - This means that in the end our exported variables are all prefixed with MODMAIL_BOT_, - and followed by the uppercase name of each field. + Required environment variables are any Config.default variables that default to marshmallow.missing + These can also be configured by using the ConfigMetadata options. """ - env_prefix = modmail.config.ENV_PREFIX + modmail.config.BOT_ENV_PREFIX default = modmail.config.get_default_config() - req_env_values: typing.Dict[str, attr.Attribute.metadata] = dict() - fields = attr.fields(default.bot.__class__) - for attribute in fields: - if attribute.default is marshmallow.missing: - req_env_values[env_prefix + attribute.name.upper()] = defaultdict(str, attribute.metadata) + + # find all environment variables to report + def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, MetadataDict]: + + if env_prefix is None: + env_prefix = modmail.config.ENV_PREFIX + + # exact name, default value + export: typing.Dict[str, MetadataDict] = dict() # any missing required vars provide as missing + + for var in attr.fields(klass): + if attr.has(var.type): + # var is an attrs class too, run this on it. + export.update( + get_env_vars( + var.type, + env_prefix=env_prefix + var.name.upper() + "_", + ) + ) + else: + meta: MetadataDict = var.metadata + # put all values in the dict, we'll iterate through them later. + export[env_prefix + var.name.upper()] = meta + + return export # dotenv modifies currently existing files, but we want to erase the current file ENV_EXPORT_FILE.unlink(missing_ok=True) ENV_EXPORT_FILE.touch() - for k, v in req_env_values.items(): - dotenv.set_key(ENV_EXPORT_FILE, k, v[modmail.config.METADATA_TABLE].export_environment_prefill) + exported = get_env_vars(default.__class__) - # the rest of this is designated for the app.json file with open(APP_JSON_FILE) as f: try: app_json: typing.Dict = json.load(f) @@ -104,19 +122,35 @@ def export_env_and_app_json_conf() -> None: "If you've made manual edits, you may want to revert them." ) raise e - app_json_env = defaultdict(str) - for env_var, meta in req_env_values.items(): - options = defaultdict( - str, - { - "description": meta[modmail.config.METADATA_TABLE].description, - "required": meta[modmail.config.METADATA_TABLE].app_json_required - or meta.get("required", False), - }, - ) - if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: - options["value"] = value - app_json_env[env_var] = options + + app_json_env = dict() + + for key, meta in exported.items(): + if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): + + dotenv.set_key( + ENV_EXPORT_FILE, + key, + meta[METADATA_TABLE].export_environment_prefill or meta["default"], + ) + + if ( + meta[METADATA_TABLE].export_to_app_json + or meta[METADATA_TABLE].export_to_env_template + or meta.get("required", False) + ): + + options = defaultdict( + str, + { + "description": meta[METADATA_TABLE].description, + "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), + }, + ) + if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: + options["value"] = value + app_json_env[key] = options + app_json["env"] = app_json_env with open(APP_JSON_FILE, "w") as f: json.dump(app_json, f, indent=4) From 014e9aac39b07cd4c9617c5d4141031f7672f442 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 22:15:25 -0400 Subject: [PATCH 33/99] tests: add config loading tests test if the cofngiruation properly reads from provided files Signed-off-by: onerandomusername --- modmail/config.py | 13 +-- pyproject.toml | 7 ++ tests/modmail/test_config.py | 150 +++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 tests/modmail/test_config.py diff --git a/modmail/config.py b/modmail/config.py index 4e6ae89d..0d69e9fc 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -18,7 +18,7 @@ try: import yaml -except ImportError: +except ImportError: # pragma: nocover yaml = None __all__ = [ @@ -38,6 +38,7 @@ "convert_to_color", "get_config", "get_default_config", + "load_env", "load_toml", "load_yaml", ] @@ -164,7 +165,7 @@ class ConfigMetadata: @description.validator def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: - if not isinstance(value, attrib.type): + if not isinstance(value, attrib.type): # pragma: no branch raise ValueError(f"description must be of {attrib.type}") from None @@ -401,7 +402,7 @@ def _build_class( return klass(**kw) -def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: +def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: """ Load a configuration dictionary from the specified env file and environment variables. @@ -494,7 +495,7 @@ def _remove_extra_values(klass: type, dit: DictT) -> DictT: return cleared_dict -def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: +def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: """ Loads a configuration from the specified files. @@ -526,8 +527,8 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: "the required dependencies are not installed." ) - if load_env: - loaded_config_dict = _load_env(existing_cfg_dict=loaded_config_dict) + if should_load_env: + loaded_config_dict = load_env(existing_cfg_dict=loaded_config_dict) # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. diff --git a/pyproject.toml b/pyproject.toml index 96395a3a..2a48bdec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,13 @@ omit = ["modmail/plugins/**.*"] addopts = "--cov --cov-report=" minversion = "6.0" testpaths = ["tests"] +filterwarnings = [ + "default", + 'ignore::DeprecationWarning:marshmallow.fields:173', + 'ignore::DeprecationWarning:marshmallow.fields:438', + 'ignore::DeprecationWarning:marshmallow.fields:456', +] + [tool.black] line-length = 110 diff --git a/tests/modmail/test_config.py b/tests/modmail/test_config.py new file mode 100644 index 00000000..2548c450 --- /dev/null +++ b/tests/modmail/test_config.py @@ -0,0 +1,150 @@ +import inspect +import os +import pathlib +import textwrap +import typing + +import atoml +import attr +import desert +import discord +import discord.ext.commands.converter +import dotenv +import marshmallow.utils +import pytest + +from modmail import config + + +def test_config_is_cached(): + """Test configuration is cached, helping keep only one version of the configuration in existance.""" + for _ in range(2): + assert config.config() == config._CACHED_CONFIG + + +def test_default_config_is_cached(): + """Test default configuration is cached, helping keep only one version of the config in existance.""" + for _ in range(2): + assert config.default() == config._CACHED_DEFAULT + + +class TestConfigLoaders: + """Test configuration loaders properly read and decode their files.""" + + def test_load_env(self, tmp_path: pathlib.Path): + """ + Ensure an environment variable properly gets loaded. + + This writes a custom .env file and loads them with dotenv. + """ + token = "NjQzOTQ1MjY0ODY4MDk4MDQ5.342b.4inDLBILY69LOLfyi6jk420dpyjoEVsCoModM" # noqa: S105 + prefix = "oop" + dev_mode = True + env = textwrap.dedent( + f""" + MODMAIL_BOT_TOKEN="{token}" + MODMAIL_BOT_PREFIX="{prefix}" + MODMAIL_DEV_MODE_DEVELOP={dev_mode} + """ + ) + test_env = tmp_path / ".env" + with open(test_env, "w") as f: + f.write(env + "\n") + + # we have to run this here since we may have the environment vars in our local env + # but we want to ensure that they are of the above env for the test + dotenv.load_dotenv(test_env, override=True) + cfg_dict = config.load_env(test_env) + + assert token == cfg_dict["bot"]["token"] + assert prefix == cfg_dict["bot"]["prefix"] + assert dev_mode == bool(cfg_dict["dev"]["mode"]["develop"]) + + def test_load_toml(self, tmp_path: pathlib.Path): + """ + Ensure a toml file is loaded is properly loaded. + + This writes a temporary file to the tempfolder and then parses it, deleting it when done. + """ + # toml is a little bit different, so we have to convert our bools to strings + # and then make them lowercase + prefix = "toml_ftw" + log_level = 40 + develop = True + plugin_dev = True + toml = textwrap.dedent( + f""" + [bot] + prefix = "{prefix}" + + [dev] + log_level = {log_level} + + [dev.mode] + develop = {str(develop).lower()} + plugin_dev = {str(plugin_dev).lower()} + """ + ) + test_toml = tmp_path / "test.toml" + with open(test_toml, "w") as f: + f.write(toml + "\n") + + cfg_dict = config.load_toml(test_toml) + + assert prefix == cfg_dict["bot"]["prefix"] + assert log_level == cfg_dict["dev"]["log_level"] + assert develop == bool(cfg_dict["dev"]["mode"]["develop"]) + assert plugin_dev == bool(cfg_dict["dev"]["mode"]["plugin_dev"]) + + def test_load_yaml(self, tmp_path: pathlib.Path): + """ + Ensure a yaml file is loaded is properly loaded. + + This writes a temporary file to the tempfolder and then parses it, deleting it when done. + """ + # this test requires the yaml library to be installed. + _ = pytest.importorskip("yaml", reason="Yaml is not installed, unable to test yaml loading.") + prefix = "toml_ftw" + log_level = 40 + develop = True + plugin_dev = True + yaml = textwrap.dedent( + f""" + bot: + prefix: '{prefix}' + dev: + log_level: {log_level} + mode: + develop: {str(develop).lower()} + plugin_dev: {str(plugin_dev).lower()} + + """ + ) + test_yaml = tmp_path / "test.yaml" + with open(test_yaml, "w") as f: + f.write(yaml + "\n") + + cfg_dict = config.load_yaml(test_yaml) + + assert prefix == cfg_dict["bot"]["prefix"] + assert log_level == cfg_dict["dev"]["log_level"] + assert develop == bool(cfg_dict["dev"]["mode"]["develop"]) + assert plugin_dev == bool(cfg_dict["dev"]["mode"]["plugin_dev"]) + + +def test_colour_conversion(): + """ + Test the discord.py converter takes all supported colours. + + Regression test. + """ + ... + + +def test_metadata_valid(): + """ + Checks that the metadata for every field is valid. + + This is more of a sanity check than anything. + """ + ... From 2a2b9dcc2131dbac0a6a525dca8c202db8c5d703 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 00:24:07 -0400 Subject: [PATCH 34/99] feat: add configuration manager cog Signed-off-by: onerandomusername --- modmail/config.py | 3 +- modmail/extensions/configuration_manager.py | 123 ++++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 modmail/extensions/configuration_manager.py diff --git a/modmail/config.py b/modmail/config.py index 0d69e9fc..78fbaf12 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -196,7 +196,7 @@ class BotCfg: }, ) prefix: str = attr.ib( - default=marshmallow.missing, + default="?", metadata={ METADATA_TABLE: ConfigMetadata( canconical_name="Command Prefix", @@ -207,7 +207,6 @@ class BotCfg: export_to_env_template=True, ) }, - converter=lambda x: "?" if x is marshmallow.missing else x, ) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py new file mode 100644 index 00000000..bed79aec --- /dev/null +++ b/modmail/extensions/configuration_manager.py @@ -0,0 +1,123 @@ +import logging +import string +import typing + +import attr +import marshmallow +from discord.ext import commands +from discord.ext.commands import Context + +from modmail import config +from modmail.bot import ModmailBot +from modmail.log import ModmailLogger +from modmail.utils.cogs import ExtMetadata, ModmailCog +from modmail.utils.pagination import ButtonPaginator + + +EXT_METADATA = ExtMetadata() + +logger: ModmailLogger = logging.getLogger(__name__) + + +@attr.mutable +class ConfOptions: + """Configuration attribute class.""" + + default: str + name: str + description: str + canconical_name: str + extended_description: str + hidden: bool + + _type: type + + nested: str = None + + @classmethod + def from_field(cls, field: attr.Attribute, nested: str = None): + """Create a ConfOptions from a attr.Attribute.""" + kw = {} + kw["default"] = field.default if field.default is not marshmallow.missing else None + kw["name"] = field.name + kw["type"] = field.type + + meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] + kw["description"] = meta.description + kw["canconical_name"] = meta.canconical_name + kw["extended_description"] = meta.extended_description + kw["hidden"] = meta.hidden + + if nested is not None: + kw["nested"] = nested + + return cls(**kw) + + +def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing.Dict[str, ConfOptions]: + """Get a dict of ConfOptions for a designated configuration field recursively.""" + options = dict() + for field in attr.fields(klass): + # make conf option list + if attr.has(field.type): + options.update(get_all_conf_options(field.type, prefix=field.name + ".")) + else: + try: + conf_opt = ConfOptions.from_field(field, nested=prefix) + except KeyError as e: + if field.type == type: + pass + elif config.METADATA_TABLE in e.args[0]: + logger.warn( + f"Issue with field '{field.name}', does not have a {config.METADATA_TABLE} key." + ) + else: + logger.error(f"A key error occured with {field.name}.", exc_info=True) + else: + options[prefix + field.name] = conf_opt + + return options + + +class ConfigurationManager(ModmailCog, name="Configuration Manager"): + """Manage the bot configuration.""" + + config_fields: typing.Dict[str, ConfOptions] + + def __init__(self, bot: ModmailBot): + self.bot = bot + + self.config_fields = get_all_conf_options(config.default().__class__) + + @commands.group(name="config", aliases=("cfg", "conf"), invoke_without_command=True) + async def config_group(self, ctx: Context) -> None: + """Manage the bot configuration.""" + if ctx.invoked_subcommand is None: + await ctx.send_help(ctx.command) + + @config_group.command(name="list") + async def list_config(self, ctx: Context) -> None: + """List the valid configuration options.""" + options = [] + for opt in self.config_fields.values(): + if opt.hidden: + continue + options.append( + "\n".join( + [ + f"**{string.capwords(opt.canconical_name or opt.name)}**", + f"Default: `{opt.default}`" + if opt.default is not None + else "Required. There is no default value for this option.", + f"{opt.description}", + f" {opt.extended_description}" if opt.extended_description else "", + ] + ) + ) + + await ButtonPaginator.paginate(options, ctx.message) + + +def setup(bot: ModmailBot) -> None: + """Load the ConfigurationManager cog.""" + bot.add_cog(ConfigurationManager(bot)) From 9eeee2e2f0e7db21da68b0a7af0fb9aa9cc4abd0 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 08:24:31 -0400 Subject: [PATCH 35/99] migrate some existing values to new configuration Signed-off-by: onerandomusername --- modmail/config.py | 43 +++++++++++++++++++++++++ modmail/default_config.toml | 4 +++ modmail/default_config.yaml | 3 ++ modmail/extensions/extension_manager.py | 25 +++++++++----- modmail/utils/embeds.py | 4 +-- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 78fbaf12..52c6b572 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -9,6 +9,7 @@ import attr import desert import discord +import discord.ext.commands import discord.ext.commands.converter import dotenv import marshmallow @@ -159,6 +160,11 @@ class ConfigMetadata: app_json_default: str = None app_json_required: bool = None + # I have no plans to add async to this file, that would make it overly complex. + # as a solution, I'm implementing a field which can provide a rich converter object, + # in the style that discord.py uses. This will be called like discord py calls. + discord_converter: discord.ext.commands.converter.Converter = attr.ib(default=None) + # hidden, eg log_level # hidden values mean they do not show up in the bot configuration menu hidden: bool = False @@ -168,6 +174,13 @@ def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> No if not isinstance(value, attrib.type): # pragma: no branch raise ValueError(f"description must be of {attrib.type}") from None + @discord_converter.validator + def _validate_discord_converter(self, attrib: attr.Attribute, value: typing.Any) -> None: + if value is None: + return + if not hasattr(value, "convert"): # pragma: no branch + raise AttributeError("Converters must have a method named convert.") + @attr.s(auto_attribs=True, slots=True) class BotCfg: @@ -294,6 +307,35 @@ def _log_level_validator(self, a: attr.Attribute, value: int) -> None: raise ValueError("log_level must be an integer within 0 to 50, inclusive.") +@attr.mutable(slots=True) +class EmojiCfg: + """ + Emojis used across the entire bot. + + This was a pain to implement. + """ + + success: typing.Any = attr.ib( + default=":thumbsup:", + metadata={ + METADATA_TABLE: ConfigMetadata( + description="This is used in most cases when the bot does a successful action.", + discord_converter=discord.ext.commands.converter.EmojiConverter, + ) + }, + ) + + failure: typing.Any = attr.ib( + default=":x:", + metadata={ + METADATA_TABLE: ConfigMetadata( + description="This is used in most cases when the bot fails an action.", + discord_converter=discord.ext.commands.converter.EmojiConverter, + ) + }, + ) + + @attr.s(auto_attribs=True, slots=True) class Cfg: """ @@ -315,6 +357,7 @@ class Cfg: ) }, ) + emojis: EmojiCfg = EmojiCfg() # build configuration diff --git a/modmail/default_config.toml b/modmail/default_config.toml index de41e516..a3202156 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -15,3 +15,7 @@ log_level = 20 develop = false plugin_dev = false production = true + +[emojis] +failure = ":x:" +success = ":thumbsup:" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index 7ada283b..de7c70cb 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -10,3 +10,6 @@ dev: develop: false plugin_dev: false production: true +emojis: + failure: ':x:' + success: ':thumbsup:' diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index dee0f01b..19554a78 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -11,6 +11,7 @@ from discord.ext import commands from discord.ext.commands import Context +import modmail.config from modmail.bot import ModmailBot from modmail.log import ModmailLogger from modmail.utils.cogs import BotModes, ExtMetadata, ModmailCog @@ -26,6 +27,9 @@ EXT_METADATA = ExtMetadata(load_if_mode=BotModes.DEVELOP, no_unload=True) +Emojis = modmail.config.config().user.emojis + + class Action(Enum): """Represents an action to perform on an extension.""" @@ -66,12 +70,12 @@ async def convert(self, _: Context, argument: str) -> str: matches.append(ext) if not matches: - raise commands.BadArgument(f":x: Could not find the {self.type} `{argument}`.") + raise commands.BadArgument(f"{Emojis.failure} Could not find the {self.type} `{argument}`.") if len(matches) > 1: names = "\n".join(sorted(matches)) raise commands.BadArgument( - f":x: `{argument}` is an ambiguous {self.type} name. " + f"{Emojis.failure} `{argument}` is an ambiguous {self.type} name. " f"Please use one of the following fully-qualified names.```\n{names}```" ) @@ -134,7 +138,9 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) if blacklisted: bl_msg = "\n".join(blacklisted) - await ctx.send(f":x: The following {self.type}(s) may not be unloaded:```\n{bl_msg}```") + await ctx.send( + f"{Emojis.failure} The following {self.type}(s) may not be unloaded:```\n{bl_msg}```" + ) return if "*" in extensions: @@ -253,7 +259,7 @@ def batch_manage(self, action: Action, *extensions: str) -> str: if error: failures[extension] = error - emoji = ":x:" if failures else ":thumbsup:" + emoji = Emojis.failure if failures else Emojis.success msg = f"{emoji} {len(extensions) - len(failures)} / {len(extensions)} {self.type}s {verb}ed." if failures: @@ -274,9 +280,12 @@ def manage(self, action: Action, ext: str) -> t.Tuple[str, t.Optional[str]]: except (commands.ExtensionAlreadyLoaded, commands.ExtensionNotLoaded): if action is Action.RELOAD: # When reloading, have a special error. - msg = f":x: {self.type.capitalize()} `{ext}` is not loaded, so it was not {verb}ed." + msg = ( + f"{Emojis.failure} {self.type.capitalize()} " + f"`{ext}` is not loaded, so it was not {verb}ed." + ) else: - msg = f":x: {self.type.capitalize()} `{ext}` is already {verb}ed." + msg = f"{Emojis.failure} {self.type.capitalize()} `{ext}` is already {verb}ed." except Exception as e: if hasattr(e, "original"): # If original exception is present, then utilize it @@ -285,9 +294,9 @@ def manage(self, action: Action, ext: str) -> t.Tuple[str, t.Optional[str]]: log.exception(f"{self.type.capitalize()} '{ext}' failed to {verb}.") error_msg = f"{e.__class__.__name__}: {e}" - msg = f":x: Failed to {verb} {self.type} `{ext}`:\n```\n{error_msg}```" + msg = f"{Emojis.failure} Failed to {verb} {self.type} `{ext}`:\n```\n{error_msg}```" else: - msg = f":thumbsup: {self.type.capitalize()} successfully {verb}ed: `{ext}`." + msg = f"{Emojis.success} {self.type.capitalize()} successfully {verb}ed: `{ext}`." log.debug(error_msg or msg) return msg, error_msg diff --git a/modmail/utils/embeds.py b/modmail/utils/embeds.py index 9db685a8..70e7b24d 100644 --- a/modmail/utils/embeds.py +++ b/modmail/utils/embeds.py @@ -6,8 +6,6 @@ from modmail.config import config -DEFAULT_COLOR = config().user.colours.base_embed_color - original_init = discord.Embed.__init__ @@ -42,7 +40,7 @@ def __init__(self: discord.Embed, description: str = None, **kwargs): # noqa: N description=description or content or EmptyEmbed, type=kwargs.pop("type", "rich"), url=kwargs.pop("url", EmptyEmbed), - colour=kwargs.pop("color", kwargs.pop("colour", DEFAULT_COLOR)), + colour=kwargs.pop("color", kwargs.pop("colour", config().user.colours.base_embed_color)), timestamp=kwargs.pop("timestamp", EmptyEmbed), ) From d6cbf869231c25eacd975fb9de6a608150ecd357 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 08:25:45 -0400 Subject: [PATCH 36/99] minor: hide frozen values which can't be changed from the configuration Signed-off-by: onerandomusername --- modmail/extensions/configuration_manager.py | 24 ++++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index bed79aec..517c06e3 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -3,6 +3,7 @@ import typing import attr +import attr._make import marshmallow from discord.ext import commands from discord.ext.commands import Context @@ -33,15 +34,18 @@ class ConfOptions: _type: type nested: str = None + frozen: bool = False @classmethod - def from_field(cls, field: attr.Attribute, nested: str = None): + def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str = None): """Create a ConfOptions from a attr.Attribute.""" kw = {} kw["default"] = field.default if field.default is not marshmallow.missing else None kw["name"] = field.name kw["type"] = field.type + kw["frozen"] = field.on_setattr is attr.setters.frozen or frozen + meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] kw["description"] = meta.description kw["canconical_name"] = meta.canconical_name @@ -62,8 +66,9 @@ def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing. if attr.has(field.type): options.update(get_all_conf_options(field.type, prefix=field.name + ".")) else: + is_frozen = klass.__setattr__ is attr._make._frozen_setattrs try: - conf_opt = ConfOptions.from_field(field, nested=prefix) + conf_opt = ConfOptions.from_field(field, frozen=is_frozen, nested=prefix) except KeyError as e: if field.type == type: pass @@ -98,11 +103,14 @@ async def config_group(self, ctx: Context) -> None: @config_group.command(name="list") async def list_config(self, ctx: Context) -> None: """List the valid configuration options.""" - options = [] - for opt in self.config_fields.values(): - if opt.hidden: + options = {} + for table, opt in self.config_fields.items(): + if opt.hidden or opt.frozen: continue - options.append( + + # we want to merge items from the same config table so they are on the same table + key = table.rsplit(".", 1)[0] + options[key] = options.get(key, "") + ( "\n".join( [ f"**{string.capwords(opt.canconical_name or opt.name)}**", @@ -110,12 +118,12 @@ async def list_config(self, ctx: Context) -> None: if opt.default is not None else "Required. There is no default value for this option.", f"{opt.description}", - f" {opt.extended_description}" if opt.extended_description else "", + f" {opt.extended_description}\n" if opt.extended_description else "", ] ) ) - await ButtonPaginator.paginate(options, ctx.message) + await ButtonPaginator.paginate(options.values(), ctx.message) def setup(bot: ModmailBot) -> None: From 249fa5eb65209b41c3b2776886297f4cbeeea1da Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 10:37:19 -0400 Subject: [PATCH 37/99] feat: rudimentary setting and getting configuration status Signed-off-by: onerandomusername --- modmail/extensions/configuration_manager.py | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 517c06e3..8e76e28e 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,4 +1,5 @@ import logging +import operator import string import typing @@ -33,6 +34,11 @@ class ConfOptions: _type: type + metadata: dict + + modmail_metadata: config.ConfigMetadata + + _field: attr.Attribute = None nested: str = None frozen: bool = False @@ -44,9 +50,13 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str kw["name"] = field.name kw["type"] = field.type + kw["metadata"] = field.metadata + kw["field"] = field + kw["frozen"] = field.on_setattr is attr.setters.frozen or frozen meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] + kw[config.METADATA_TABLE] = meta kw["description"] = meta.description kw["canconical_name"] = meta.canconical_name kw["extended_description"] = meta.extended_description @@ -125,6 +135,35 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message) + @config_group.command(name="set", aliases=("edit",)) + async def modify_config(self, ctx: Context, option: str, value: str) -> None: + """Modify an existing configuration value.""" + if option not in self.config_fields: + raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") + meta = self.config_fields[option] + + if meta.frozen: + await ctx.send("Can't modify this value.") + return + + if meta.modmail_metadata.discord_converter is not None: + value = await meta.modmail_metadata.discord_converter().convert(ctx, value) + elif meta._field.converter: + value = meta._field.converter(value) + get_value = operator.attrgetter(option.rsplit(".", -1)[0]) + setattr(get_value(self.bot.config.user), option.rsplit(".", -1)[-1], value) + await ctx.message.reply("ok.") + + @config_group.command(name="get", aliases=("show",)) + async def get_config(self, ctx: Context, option: str) -> None: + """Modify an existing configuration value.""" + if option not in self.config_fields: + raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") + + get_value = operator.attrgetter(option) + value = get_value(self.bot.config.user) + await ctx.send(f"{option}: `{value}`") + def setup(bot: ModmailBot) -> None: """Load the ConfigurationManager cog.""" From a3402a8c399e8a4f2b58d62203c55a811b1767c9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 18:07:46 -0400 Subject: [PATCH 38/99] refactor getvalue Signed-off-by: onerandomusername --- modmail/extensions/configuration_manager.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 8e76e28e..542cfadb 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -150,8 +150,15 @@ async def modify_config(self, ctx: Context, option: str, value: str) -> None: value = await meta.modmail_metadata.discord_converter().convert(ctx, value) elif meta._field.converter: value = meta._field.converter(value) - get_value = operator.attrgetter(option.rsplit(".", -1)[0]) - setattr(get_value(self.bot.config.user), option.rsplit(".", -1)[-1], value) + + try: + root, name = option.rsplit(".", -1) + except ValueError: + root = "" + name = option + + setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + await ctx.message.reply("ok.") @config_group.command(name="get", aliases=("show",)) From 21a46d4e69ba4713a53b8ca20d95729fffea7313 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 24 Sep 2021 06:47:30 -0400 Subject: [PATCH 39/99] fix: don't load the .env file to the environment Signed-off-by: onerandomusername --- modmail/config.py | 6 ++++-- tests/modmail/test_config.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 52c6b572..f058284b 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -409,8 +409,10 @@ def _build_class( if env is None: if dotenv_file is not None: - dotenv.load_dotenv(dotenv_file) - env = os.environ.copy() + env = dotenv.dotenv_values(dotenv_file) + env.update(os.environ) + else: + env = os.environ.copy() # get the attributes of the provided class if defaults is None: diff --git a/tests/modmail/test_config.py b/tests/modmail/test_config.py index 2548c450..a7a580a8 100644 --- a/tests/modmail/test_config.py +++ b/tests/modmail/test_config.py @@ -53,8 +53,16 @@ def test_load_env(self, tmp_path: pathlib.Path): # we have to run this here since we may have the environment vars in our local env # but we want to ensure that they are of the above env for the test - dotenv.load_dotenv(test_env, override=True) - cfg_dict = config.load_env(test_env) + try: + os.environ.update(dotenv.dotenv_values(test_env)) + cfg_dict = config.load_env(test_env) + finally: + # clean up the loaded values + for key in dotenv.dotenv_values(test_env): + try: + del os.environ[key] + except ValueError: + pass assert token == cfg_dict["bot"]["token"] assert prefix == cfg_dict["bot"]["prefix"] From a22f51a4d1e105c711208e2cfed89d9475a6be8d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 24 Sep 2021 09:23:34 -0400 Subject: [PATCH 40/99] minor: change config pre-commit hook to exit with 1 if files edited Signed-off-by: onerandomusername --- .../export_new_config_to_default_config.py | 178 +++++++++++------- 1 file changed, 114 insertions(+), 64 deletions(-) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index d3098400..86db9b75 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -4,6 +4,7 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ import json +import os import pathlib import sys import typing @@ -14,6 +15,7 @@ import dotenv import yaml +import modmail import modmail.config @@ -22,6 +24,7 @@ APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" METADATA_TABLE = modmail.config.METADATA_TABLE +MODMAIL_DIR = pathlib.Path(modmail.__file__).parent class MetadataDict(typing.TypedDict): @@ -31,7 +34,34 @@ class MetadataDict(typing.TypedDict): required: bool -def export_default_conf() -> None: +class DidFileEdit: + """Check if a file is edited within the body of this class.""" + + def __init__(self, *files: os.PathLike): + self.files: typing.List[os.PathLike] = [] + for f in files: + self.files.append(f) + self.return_value: typing.Optional[int] = None + self.edited_files: typing.List[os.PathLike] = [] + + def __enter__(self): + self.file_contents = {} + for file in self.files: + try: + with open(file, "r") as f: + self.file_contents[file] = f.read() + except FileNotFoundError: + self.file_contents[file] = None + + def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 + for file in self.files: + with open(file, "r") as f: + contents = self.file_contents[file] + if contents != f.read(): + self.edited_files.append(file) + + +def export_default_conf() -> int: """Export default configuration as both toml and yaml to the preconfigured locations.""" default = modmail.config.get_default_config() dump: dict = modmail.config.ConfigurationSchema().dump(default) @@ -58,19 +88,29 @@ def sort_dict(d: dict) -> dict: doc.update(dump) - # toml + toml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml") + yaml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml") + + check_file = DidFileEdit(toml_file, yaml_file) - with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml"), "w") as f: - atoml.dump(doc, f) + with check_file: + with open(toml_file, "w") as f: + atoml.dump(doc, f) - # yaml - with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: - f.write("# This is an autogenerated YAML document.\n") - f.write(f"# {autogen_gen_notice}\n") - yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + with open(yaml_file, "w") as f: + f.write("# This is an autogenerated YAML document.\n") + f.write(f"# {autogen_gen_notice}\n") + yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + for file in check_file.edited_files: + print( + f"Exported new configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", + file=sys.stderr, + ) + return bool(len(check_file.edited_files)) -def export_env_and_app_json_conf() -> None: + +def export_env_and_app_json_conf() -> int: """ Exports required configuration variables to .env.template. @@ -107,57 +147,68 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada return export - # dotenv modifies currently existing files, but we want to erase the current file - ENV_EXPORT_FILE.unlink(missing_ok=True) - ENV_EXPORT_FILE.touch() - - exported = get_env_vars(default.__class__) - - with open(APP_JSON_FILE) as f: - try: - app_json: typing.Dict = json.load(f) - except Exception as e: - print( - "Oops! Please ensure the app.json file is valid json! " - "If you've made manual edits, you may want to revert them." - ) - raise e - - app_json_env = dict() - - for key, meta in exported.items(): - if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): - - dotenv.set_key( - ENV_EXPORT_FILE, - key, - meta[METADATA_TABLE].export_environment_prefill or meta["default"], - ) - - if ( - meta[METADATA_TABLE].export_to_app_json - or meta[METADATA_TABLE].export_to_env_template - or meta.get("required", False) - ): - - options = defaultdict( - str, - { - "description": meta[METADATA_TABLE].description, - "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), - }, - ) - if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: - options["value"] = value - app_json_env[key] = options - - app_json["env"] = app_json_env - with open(APP_JSON_FILE, "w") as f: - json.dump(app_json, f, indent=4) - f.write("\n") - - -def main() -> None: + check_file = DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) + + with check_file: + # dotenv modifies currently existing files, but we want to erase the current file + ENV_EXPORT_FILE.unlink(missing_ok=True) + ENV_EXPORT_FILE.touch() + + exported = get_env_vars(default.__class__) + + with open(APP_JSON_FILE) as f: + try: + app_json: typing.Dict = json.load(f) + except Exception as e: + print( + "Oops! Please ensure the app.json file is valid json! " + "If you've made manual edits, you may want to revert them.", + file=sys.stderr, + ) + raise e + + app_json_env = dict() + + for key, meta in exported.items(): + if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): + + dotenv.set_key( + ENV_EXPORT_FILE, + key, + meta[METADATA_TABLE].export_environment_prefill or meta["default"], + ) + + if ( + meta[METADATA_TABLE].export_to_app_json + or meta[METADATA_TABLE].export_to_env_template + or meta.get("required", False) + ): + + options = defaultdict( + str, + { + "description": meta[METADATA_TABLE].description, + "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), + }, + ) + if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: + options["value"] = value + app_json_env[key] = options + + app_json["env"] = app_json_env + with open(APP_JSON_FILE, "w") as f: + json.dump(app_json, f, indent=4) + f.write("\n") + + for file in check_file.edited_files: + print( + f"Exported new env configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", + file=sys.stderr, + ) + return bool(len(check_file.edited_files)) + + +def main() -> int: """ Exports the default configuration. @@ -168,11 +219,10 @@ def main() -> None: In addition, export to app.json when exporting .env.template. """ - export_default_conf() + exit_code = export_default_conf() - export_env_and_app_json_conf() + return export_env_and_app_json_conf() or exit_code if __name__ == "__main__": - print("Exporting configuration to default files. If they exist, overwriting their contents.") sys.exit(main()) From 1c7dd19b49ffe3e3b946d44fc46602572f8bc968 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 24 Sep 2021 23:49:02 -0400 Subject: [PATCH 41/99] fix precommit requirements Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed670ebc..a51e17b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,15 +11,17 @@ repos: files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' require_serial: true additional_dependencies: - - atoml~=1.0.3 - - attrs~=21.2.0 + # so apparently these are needed, but the versions don't have to be pinned since it uses the local env + # go figure. + - atoml + - attrs - coloredlogs - desert - discord.py - environs - - marshmallow~=3.13.0 + - marshmallow - python-dotenv - - pyyaml~=5.4.1 + - pyyaml - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 From dd0e28785ef4681cfed7bc38c2c34e35d0e9ff2a Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 25 Sep 2021 00:42:11 -0400 Subject: [PATCH 42/99] config: allow emojis to be set (needs more work) Signed-off-by: onerandomusername --- modmail/config.py | 34 +++++++++++++- modmail/extensions/configuration_manager.py | 52 +++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index f058284b..6b81f743 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -2,6 +2,7 @@ import logging import os import pathlib +import types import typing from collections import defaultdict @@ -54,6 +55,30 @@ _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] + +class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): + """ + Converts to a :class:`~discord.PartialEmoji`. + + This is done by extracting the animated flag, name and ID from the emoji. + """ + + async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: + # match = self._get_id_match(argument) or re.match( + # r"$", argument + # ) + + match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) + if match is not None: + groups = match.groupdict() + animated = bool(groups["animated"]) + emoji_id = int(groups["id"]) + name = groups["name"] + return discord.PartialEmoji(name=name, animated=animated, id=emoji_id) + + return discord.PartialEmoji(name=argument) + + # load env before we do *anything* # TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. # TODO: add the above feature to the other configuration locations too. @@ -164,6 +189,9 @@ class ConfigMetadata: # as a solution, I'm implementing a field which can provide a rich converter object, # in the style that discord.py uses. This will be called like discord py calls. discord_converter: discord.ext.commands.converter.Converter = attr.ib(default=None) + discord_converter_attribute: typing.Optional[ + types.FunctionType + ] = None # if we want an attribute off of the converted value # hidden, eg log_level # hidden values mean they do not show up in the bot configuration menu @@ -320,7 +348,8 @@ class EmojiCfg: metadata={ METADATA_TABLE: ConfigMetadata( description="This is used in most cases when the bot does a successful action.", - discord_converter=discord.ext.commands.converter.EmojiConverter, + discord_converter=BetterPartialEmojiConverter, + discord_converter_attribute=lambda x: x.id or f"{x.name}", ) }, ) @@ -330,7 +359,8 @@ class EmojiCfg: metadata={ METADATA_TABLE: ConfigMetadata( description="This is used in most cases when the bot fails an action.", - discord_converter=discord.ext.commands.converter.EmojiConverter, + discord_converter=BetterPartialEmojiConverter, + discord_converter_attribute=lambda x: x.id or f"{x.name}", ) }, ) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 542cfadb..feacbbaa 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,6 +1,7 @@ import logging import operator import string +import types import typing import attr @@ -8,6 +9,7 @@ import marshmallow from discord.ext import commands from discord.ext.commands import Context +from discord.ext.commands import converter as commands_converter from modmail import config from modmail.bot import ModmailBot @@ -20,6 +22,8 @@ logger: ModmailLogger = logging.getLogger(__name__) +KeyT = str + @attr.mutable class ConfOptions: @@ -68,13 +72,13 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str return cls(**kw) -def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing.Dict[str, ConfOptions]: +def get_all_conf_options(klass: config.ClassT, *, prefix: str = "") -> typing.Dict[str, ConfOptions]: """Get a dict of ConfOptions for a designated configuration field recursively.""" options = dict() for field in attr.fields(klass): # make conf option list if attr.has(field.type): - options.update(get_all_conf_options(field.type, prefix=field.name + ".")) + options.update(get_all_conf_options(field.type, prefix=prefix + field.name + ".")) else: is_frozen = klass.__setattr__ is attr._make._frozen_setattrs try: @@ -94,6 +98,38 @@ def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing. return options +class KeyConverter(commands.Converter): + """Convert argument into a configuration key.""" + + async def convert(self, ctx: Context, arg: str) -> KeyT: + """Ensure that a key is of the valid format, allowing a user to input other formats.""" + # basically we're converting an argument to a key. + # config keys are delimited by `.`, and always lowercase, which means that we can make a few passes + # before actually trying to make any guesses. + # the problems are twofold. a: we want to suggest other keys + # and b: the interface needs to be easy to use. + + # as a partial solution for this, `/`, `-`, `.` are all valid delimiters and are converted to `.` + # depending on common problems, it is *possible* to add `_` but would require fuzzy matching over + # all of the keys since that can also be a valid character name. + + fields = get_all_conf_options(config.default().__class__) + + new_arg = "" + for c in arg.lower(): + if c in " /`": + new_arg += "." + else: + new_arg += c + + if new_arg in fields: + return new_arg + else: + raise commands.BadArgument( + f"{ctx.current_parameter.name} {arg} is not a valid configuration key." + ) + + class ConfigurationManager(ModmailCog, name="Configuration Manager"): """Manage the bot configuration.""" @@ -136,18 +172,26 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message) @config_group.command(name="set", aliases=("edit",)) - async def modify_config(self, ctx: Context, option: str, value: str) -> None: + async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" if option not in self.config_fields: raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") meta = self.config_fields[option] if meta.frozen: + # TODO: replace with responses module. await ctx.send("Can't modify this value.") return if meta.modmail_metadata.discord_converter is not None: value = await meta.modmail_metadata.discord_converter().convert(ctx, value) + if meta.modmail_metadata.discord_converter_attribute is not None: + if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): + value = meta.modmail_metadata.discord_converter_attribute(value) + if meta._type in commands_converter.CONVERTER_MAPPING: + value = await commands_converter.CONVERTER_MAPPING[meta._type]().convert(ctx, value) + if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): + value = meta.modmail_metadata.discord_converter_attribute(value) elif meta._field.converter: value = meta._field.converter(value) @@ -162,7 +206,7 @@ async def modify_config(self, ctx: Context, option: str, value: str) -> None: await ctx.message.reply("ok.") @config_group.command(name="get", aliases=("show",)) - async def get_config(self, ctx: Context, option: str) -> None: + async def get_config(self, ctx: Context, option: KeyConverter) -> None: """Modify an existing configuration value.""" if option not in self.config_fields: raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") From 76ff3cd6363711bc4acaacbc038a73b8a5717689 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 4 Oct 2021 01:15:29 -0400 Subject: [PATCH 43/99] nit: fix typo in variable name --- modmail/config.py | 8 ++++---- modmail/extensions/configuration_manager.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 6b81f743..8725d3a0 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -169,7 +169,7 @@ class ConfigMetadata: # what to refer to for the end user description: str = attr.ib() - canconical_name: str = None + canonical_name: str = None # for those configuration options where the description just won't cut it. extended_description: str = None # for the variables to export to the environment, what value should be prefilled @@ -228,7 +228,7 @@ class BotCfg: "dump_only": True, "allow_none": False, METADATA_TABLE: ConfigMetadata( - canconical_name="Bot Token", + canonical_name="Bot Token", description="Discord bot token. Required to log in to discord.", export_environment_prefill="Bot Token", extended_description="This is obtainable from https://discord.com/developers/applications", @@ -240,7 +240,7 @@ class BotCfg: default="?", metadata={ METADATA_TABLE: ConfigMetadata( - canconical_name="Command Prefix", + canonical_name="Command Prefix", description="Command prefix.", export_environment_prefill="?", app_json_default="?", @@ -257,7 +257,7 @@ class BotModeCfg: The three bot modes for the bot. Enabling some of these may enable other bot features. `production` is used internally and is always True. - `develop` enables additonal features which are useful for bot developers. + `develop` enables additional features which are useful for bot developers. `plugin_dev` enables additional commands which are useful when working with plugins. """ diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index feacbbaa..872f1259 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -32,7 +32,7 @@ class ConfOptions: default: str name: str description: str - canconical_name: str + canonical_name: str extended_description: str hidden: bool @@ -62,7 +62,7 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] kw[config.METADATA_TABLE] = meta kw["description"] = meta.description - kw["canconical_name"] = meta.canconical_name + kw["canonical_name"] = meta.canonical_name kw["extended_description"] = meta.extended_description kw["hidden"] = meta.hidden @@ -159,7 +159,7 @@ async def list_config(self, ctx: Context) -> None: options[key] = options.get(key, "") + ( "\n".join( [ - f"**{string.capwords(opt.canconical_name or opt.name)}**", + f"**{string.capwords(opt.canonical_name or opt.name)}**", f"Default: `{opt.default}`" if opt.default is not None else "Required. There is no default value for this option.", From 940942168e7dc6b4470cf761c8e8cba7f8e7465a Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 4 Oct 2021 01:34:13 -0400 Subject: [PATCH 44/99] nit: typing, formatting, and comment changes --- modmail/config.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 8725d3a0..87ef04d0 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -134,7 +134,7 @@ def convert(self, argument: typing.Union[str, int, discord.Colour]) -> discord.C raise discord.ext.commands.converter.BadColourArgument(arg) return method() - def _serialize(self, value: discord.Colour, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: + def _serialize(self, value: discord.Colour, attr: str, obj: typing.Any, **kwargs) -> str: return "#" + hex(value.value)[2:].lower() def _deserialize( @@ -143,7 +143,7 @@ def _deserialize( attr: typing.Optional[str], data: typing.Optional[typing.Mapping[str, typing.Any]], **kwargs, - ) -> str: + ) -> discord.Colour: if not isinstance(value, discord.Colour): value = self.ColourConvert().convert(value) return value @@ -381,8 +381,9 @@ class Cfg: default=DevCfg(), metadata={ METADATA_TABLE: ConfigMetadata( - description="Developer configuration. " - "Only change these values if you know what you're doing.", + description=( + "Developer configuration. Only change these values if you know what you're doing." + ), hidden=True, ) }, @@ -432,7 +433,7 @@ def _build_class( Defaults to getting the environment variables with dotenv. Also can parse from a provided dictionary of environment variables. - If defaults is provided, uses a value from there if the environment variable is not set or is None. + If `defaults` is provided, uses a value from there if the environment variable is not set or is None. """ if env_prefix is None: env_prefix = ENV_PREFIX @@ -464,9 +465,7 @@ def _build_class( else: kw[var.name] = env.get(env_prefix + var.name.upper(), None) if kw[var.name] is None: - if defaults is not None and ( - (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing - ): + if (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing: kw[var.name] = defaults[var.name] elif var.default is not attr.NOTHING: # check for var default kw[var.name] = var.default From 627e650a520b9c30e992d0605f3f2bef348ffcd6 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 4 Oct 2021 02:08:18 -0400 Subject: [PATCH 45/99] fix: make errors more helpful if config dependencies are not installed --- modmail/config.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 87ef04d0..7ec1adaa 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -6,7 +6,6 @@ import typing from collections import defaultdict -import atoml import attr import desert import discord @@ -18,9 +17,13 @@ import marshmallow.validate +try: + import atoml +except ModuleNotFoundError: # pragma: nocover + atoml = None try: import yaml -except ImportError: # pragma: nocover +except ModuleNotFoundError: # pragma: nocover yaml = None __all__ = [ @@ -577,6 +580,13 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: Supported file types are .toml or .yaml """ + + def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn: + raise CfgLoadError( + f"The required dependency for reading {file_type} configuration files is not installed. " + f"Please install {dependency or file_type} to allow reading these files." + ) + if len(files) == 0: files = USER_CONFIG_FILES @@ -588,17 +598,18 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: # file does not exist continue - if file.suffix == ".toml" and atoml is not None: + if file.suffix == ".toml": + if atoml is None: + raise_missing_dep("toml", "atoml") loaded_config_dict = load_toml(file) break - elif file.suffix == ".yaml" and yaml is not None: + elif file.suffix == ".yaml": + if yaml is None: + raise_missing_dep("yaml", "pyyaml") loaded_config_dict = load_yaml(file) break else: - raise Exception( - "Provided configuration file is not of a supported type or " - "the required dependencies are not installed." - ) + raise CfgLoadError("Provided configuration file is not of a supported type.") if should_load_env: loaded_config_dict = load_env(existing_cfg_dict=loaded_config_dict) From 94b14e02b930c0a6ed15e9aa1a2e48bc6270911d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 7 Nov 2021 01:40:54 -0400 Subject: [PATCH 46/99] scripts: show diff when exporting config edits files --- .../export_new_config_to_default_config.py | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 86db9b75..9edef536 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -3,6 +3,7 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ +import difflib import json import os import pathlib @@ -42,23 +43,29 @@ def __init__(self, *files: os.PathLike): for f in files: self.files.append(f) self.return_value: typing.Optional[int] = None - self.edited_files: typing.List[os.PathLike] = [] + self.edited_files: typing.Dict[os.PathLike] = dict() def __enter__(self): self.file_contents = {} for file in self.files: try: with open(file, "r") as f: - self.file_contents[file] = f.read() + self.file_contents[file] = f.readlines() except FileNotFoundError: self.file_contents[file] = None + return self def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 for file in self.files: with open(file, "r") as f: - contents = self.file_contents[file] - if contents != f.read(): - self.edited_files.append(file) + original_contents = self.file_contents[file] + new_contents = f.readlines() + if original_contents != new_contents: + # construct a diff + diff = difflib.unified_diff( + original_contents, new_contents, fromfile="before", tofile="after" + ) + self.edited_files[file] = diff def export_default_conf() -> int: @@ -91,9 +98,7 @@ def sort_dict(d: dict) -> dict: toml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml") yaml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml") - check_file = DidFileEdit(toml_file, yaml_file) - - with check_file: + with DidFileEdit(toml_file, yaml_file) as check_file: with open(toml_file, "w") as f: atoml.dump(doc, f) @@ -102,11 +107,16 @@ def sort_dict(d: dict) -> dict: f.write(f"# {autogen_gen_notice}\n") yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) - for file in check_file.edited_files: + for file, diff in check_file.edited_files.items(): print( f"Exported new configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) + try: + print("".join(diff)) + except TypeError: + print("No diff to show.") + print() return bool(len(check_file.edited_files)) @@ -147,9 +157,7 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada return export - check_file = DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) - - with check_file: + with DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) as check_file: # dotenv modifies currently existing files, but we want to erase the current file ENV_EXPORT_FILE.unlink(missing_ok=True) ENV_EXPORT_FILE.touch() @@ -200,11 +208,16 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada json.dump(app_json, f, indent=4) f.write("\n") - for file in check_file.edited_files: + for file, diff in check_file.edited_files.items(): print( f"Exported new env configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) + try: + print("".join(diff)) + except TypeError: + print("No diff to show.") + print() return bool(len(check_file.edited_files)) From 4117dccf4c66cca6947722e2bf0e1a2ae4499dc4 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 7 Nov 2021 17:12:02 -0500 Subject: [PATCH 47/99] fix: no more weird new lines --- modmail/default_config.toml | 3 +-- modmail/default_config.yaml | 2 +- .../export_new_config_to_default_config.py | 20 ++++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/modmail/default_config.toml b/modmail/default_config.toml index a3202156..d77c245e 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,6 +1,5 @@ # This is an autogenerated TOML document. -# Directly run scripts/export_new_config_to_default_config.py to generate. - +# Run scripts/export_new_config_to_default_config.py as a module to generate. [bot] prefix = "?" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index de7c70cb..fd8571f1 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -1,5 +1,5 @@ # This is an autogenerated YAML document. -# Directly run scripts/export_new_config_to_default_config.py to generate. +# Run scripts/export_new_config_to_default_config.py as a module to generate. bot: prefix: '?' colours: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 9edef536..c66112f8 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -8,6 +8,7 @@ import os import pathlib import sys +import textwrap import typing from collections import defaultdict @@ -27,6 +28,15 @@ METADATA_TABLE = modmail.config.METADATA_TABLE MODMAIL_DIR = pathlib.Path(modmail.__file__).parent +# fmt: off +MESSAGE = textwrap.indent(textwrap.dedent( + f""" + This is an autogenerated {{file_type}} document. + Run scripts/{__file__.rsplit('/',1)[-1]!s} as a module to generate. + """ +), "# ").strip() + "\n" +# fmt: on + class MetadataDict(typing.TypedDict): """Typed metadata. This has a possible risk given that the modmail_metadata variable is defined.""" @@ -87,12 +97,7 @@ def sort_dict(d: dict) -> dict: return sorted_dict dump = sort_dict(dump) - autogen_gen_notice = f"Directly run scripts/{__file__.rsplit('/',1)[-1]!s} to generate." doc = atoml.document() - doc.add(atoml.comment("This is an autogenerated TOML document.")) - doc.add(atoml.comment(autogen_gen_notice)) - doc.add(atoml.nl()) - doc.update(dump) toml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml") @@ -100,11 +105,12 @@ def sort_dict(d: dict) -> dict: with DidFileEdit(toml_file, yaml_file) as check_file: with open(toml_file, "w") as f: + f.write(MESSAGE.format(file_type="TOML")) + f.write("\n") atoml.dump(doc, f) with open(yaml_file, "w") as f: - f.write("# This is an autogenerated YAML document.\n") - f.write(f"# {autogen_gen_notice}\n") + f.write(MESSAGE.format(file_type="YAML")) yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) for file, diff in check_file.edited_files.items(): From 003177e71d74f79d50840b75382cca83bf1008eb Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 19:51:39 -0500 Subject: [PATCH 48/99] minor(config): touch up the visual UI --- modmail/extensions/configuration_manager.py | 48 ++++++++++++--------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 872f1259..076d2596 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -6,6 +6,7 @@ import attr import attr._make +import discord import marshmallow from discord.ext import commands from discord.ext.commands import Context @@ -14,6 +15,7 @@ from modmail import config from modmail.bot import ModmailBot from modmail.log import ModmailLogger +from modmail.utils import responses from modmail.utils.cogs import ExtMetadata, ModmailCog from modmail.utils.pagination import ButtonPaginator @@ -48,7 +50,7 @@ class ConfOptions: @classmethod def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str = None): - """Create a ConfOptions from a attr.Attribute.""" + """Create a ConfOptions from an attr.Attribute.""" kw = {} kw["default"] = field.default if field.default is not marshmallow.missing else None kw["name"] = field.name @@ -126,7 +128,7 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: return new_arg else: raise commands.BadArgument( - f"{ctx.current_parameter.name} {arg} is not a valid configuration key." + f"{ctx.current_parameter.name.capitalize()} `{arg}` is not a valid configuration key." ) @@ -150,26 +152,30 @@ async def config_group(self, ctx: Context) -> None: async def list_config(self, ctx: Context) -> None: """List the valid configuration options.""" options = {} + + embed = discord.Embed(title="Configuration Options") for table, opt in self.config_fields.items(): + # TODO: add flag to skip this check if opt.hidden or opt.frozen: + # bot refuses to modify a hidden value + # and bot cannot modify a frozen value continue - # we want to merge items from the same config table so they are on the same table + # we want to merge items from the same config table so they display on the same pagination page + # for example, all emoji configuration options will not be split up key = table.rsplit(".", 1)[0] - options[key] = options.get(key, "") + ( - "\n".join( - [ - f"**{string.capwords(opt.canonical_name or opt.name)}**", - f"Default: `{opt.default}`" - if opt.default is not None - else "Required. There is no default value for this option.", - f"{opt.description}", - f" {opt.extended_description}\n" if opt.extended_description else "", - ] - ) - ) + if options.get(key) is None: + options[key] = f"**__{string.capwords(key)} config__**\n" + + name = opt.canonical_name or opt.name + default = f"Default: `{opt.default}`" if opt.default is not None else "Required." + description = opt.description + if opt.extended_description: + description += "\n" + opt.extended_description - await ButtonPaginator.paginate(options.values(), ctx.message) + options[key] += "\n".join([f"**{name}**", default, description]).strip() + "\n" + + await ButtonPaginator.paginate(options.values(), ctx.message, embed=embed) @config_group.command(name="set", aliases=("edit",)) async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: @@ -179,8 +185,8 @@ async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> meta = self.config_fields[option] if meta.frozen: - # TODO: replace with responses module. - await ctx.send("Can't modify this value.") + + await responses.send_negatory_response(ctx, f"Unable to modify {option}.") return if meta.modmail_metadata.discord_converter is not None: @@ -195,15 +201,15 @@ async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> elif meta._field.converter: value = meta._field.converter(value) - try: + if "." in option: root, name = option.rsplit(".", -1) - except ValueError: + else: root = "" name = option setattr(operator.attrgetter(root)(self.bot.config.user), name, value) - await ctx.message.reply("ok.") + await responses.send_positive_response(ctx, f"Successfully set `{option}` to `{value}`.") @config_group.command(name="get", aliases=("show",)) async def get_config(self, ctx: Context, option: KeyConverter) -> None: From b4e0b7d778614a4ac631b1735473e4df37b2b4ed Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 20:18:54 -0500 Subject: [PATCH 49/99] fix: update prefix when configured prefix is updated --- modmail/bot.py | 11 ++++++++++- modmail/config.py | 6 ++++++ modmail/default_config.toml | 1 + modmail/default_config.yaml | 1 + 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modmail/bot.py b/modmail/bot.py index 19f7039e..cb6dc84c 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -48,7 +48,7 @@ def __init__(self, **kwargs): activity = Activity(type=discord.ActivityType.listening, name="users dming me!") # listen to messages mentioning the bot or matching the prefix # ! NOTE: This needs to use the configuration system to get the prefix from the db once it exists. - prefix = commands.when_mentioned_or(self.config.user.bot.prefix) + prefix = self.determine_prefix # allow only user mentions by default. # ! NOTE: This may change in the future to allow roles as well allowed_mentions = AllowedMentions(everyone=False, users=True, roles=False, replied_user=True) @@ -65,6 +65,15 @@ def __init__(self, **kwargs): **kwargs, ) + @staticmethod + async def determine_prefix(bot: "ModmailBot", message: discord.Message) -> t.List[str]: + """Dynamically get the updated prefix on every command.""" + prefixes = [] + if bot.config.user.bot.prefix_when_mentioned: + prefixes.extend(commands.when_mentioned(bot, message)) + prefixes.append(bot.config.user.bot.prefix) + return prefixes + async def start(self, token: str, reconnect: bool = True) -> None: """ Start the bot. diff --git a/modmail/config.py b/modmail/config.py index 7ec1adaa..7a0f24b4 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -252,6 +252,12 @@ class BotCfg: ) }, ) + prefix_when_mentioned: bool = attr.ib( + default=True, + metadata={ + METADATA_TABLE: ConfigMetadata(description="Use the bot mention as a prefix."), + }, + ) @attr.s(auto_attribs=True, slots=True, frozen=True) diff --git a/modmail/default_config.toml b/modmail/default_config.toml index d77c245e..b1bf0b41 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -3,6 +3,7 @@ [bot] prefix = "?" +prefix_when_mentioned = true [colours] base_embed_color = "#7289da" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index fd8571f1..66e852b8 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -2,6 +2,7 @@ # Run scripts/export_new_config_to_default_config.py as a module to generate. bot: prefix: '?' + prefix_when_mentioned: true colours: base_embed_color: '#7289da' dev: From d644a96184dcbde88fea6a69a06ea3a36d355b7d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 23:45:18 -0500 Subject: [PATCH 50/99] minor fixes to export config script --- .pre-commit-config.yaml | 1 + modmail/default_config.toml | 2 +- modmail/default_config.yaml | 2 +- .../export_new_config_to_default_config.py | 64 +++++++++++++------ 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da13a94d..41e0d9ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,7 @@ repos: # go figure. - atoml - attrs + - click - coloredlogs - desert - discord.py diff --git a/modmail/default_config.toml b/modmail/default_config.toml index b1bf0b41..56f16779 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,5 +1,5 @@ # This is an autogenerated TOML document. -# Run scripts/export_new_config_to_default_config.py as a module to generate. +# Run module 'scripts.export_new_config_to_default_config' to generate. [bot] prefix = "?" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index 66e852b8..eb88ad23 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -1,5 +1,5 @@ # This is an autogenerated YAML document. -# Run scripts/export_new_config_to_default_config.py as a module to generate. +# Run module 'scripts.export_new_config_to_default_config' to generate. bot: prefix: '?' prefix_when_mentioned: true diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index c66112f8..1190000a 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -1,7 +1,7 @@ """ Exports the configuration to the configuration default files. -This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. +This is intended to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ import difflib import json @@ -14,9 +14,19 @@ import atoml import attr +import click import dotenv import yaml + +try: + import pygments +except ModuleNotFoundError: + pygments = None +else: + from pygments.lexers.diff import DiffLexer + from pygments.formatters import Terminal256Formatter + import modmail import modmail.config @@ -28,11 +38,23 @@ METADATA_TABLE = modmail.config.METADATA_TABLE MODMAIL_DIR = pathlib.Path(modmail.__file__).parent + +def get_name(path: str) -> str: + """Get the script module name of the provided file path.""" + p = pathlib.Path(__file__) + p = p.relative_to(MODMAIL_DIR.parent) + name = str(p) + if name.endswith(p.suffix): + name = name[: -len(p.suffix)] + name = name.replace("/", ".") + return name + + # fmt: off MESSAGE = textwrap.indent(textwrap.dedent( f""" This is an autogenerated {{file_type}} document. - Run scripts/{__file__.rsplit('/',1)[-1]!s} as a module to generate. + Run module '{get_name(__file__)}' to generate. """ ), "# ").strip() + "\n" # fmt: on @@ -75,6 +97,9 @@ def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 diff = difflib.unified_diff( original_contents, new_contents, fromfile="before", tofile="after" ) + diff = "".join(diff) + if diff is not None and pygments is not None: + diff = pygments.highlight(diff, DiffLexer(), Terminal256Formatter()) self.edited_files[file] = diff @@ -118,10 +143,10 @@ def sort_dict(d: dict) -> dict: f"Exported new configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) - try: - print("".join(diff)) - except TypeError: - print("No diff to show.") + if diff is not None: + click.echo(diff) + else: + click.echo("No diff to show.") print() return bool(len(check_file.edited_files)) @@ -145,11 +170,11 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada env_prefix = modmail.config.ENV_PREFIX # exact name, default value - export: typing.Dict[str, MetadataDict] = dict() # any missing required vars provide as missing + export: typing.Dict[str, MetadataDict] = dict() # any missing required vars provide a sentinel for var in attr.fields(klass): if attr.has(var.type): - # var is an attrs class too, run this on it. + # var is an attrs class too, recurse over it export.update( get_env_vars( var.type, @@ -164,12 +189,7 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada return export with DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) as check_file: - # dotenv modifies currently existing files, but we want to erase the current file - ENV_EXPORT_FILE.unlink(missing_ok=True) - ENV_EXPORT_FILE.touch() - - exported = get_env_vars(default.__class__) - + # parse the app_json file before clearing the ENV file with open(APP_JSON_FILE) as f: try: app_json: typing.Dict = json.load(f) @@ -181,9 +201,17 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada ) raise e + # dotenv modifies currently existing files, but we want to erase the current file + # to ensure that there are no extra environment variables + ENV_EXPORT_FILE.unlink(missing_ok=True) + ENV_EXPORT_FILE.touch() + + exported = get_env_vars(default.__class__) + app_json_env = dict() for key, meta in exported.items(): + # if the value is required, or explicity asks to be exported, then we want to export it if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): dotenv.set_key( @@ -219,10 +247,10 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada f"Exported new env configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) - try: - print("".join(diff)) - except TypeError: - print("No diff to show.") + if diff is not None: + click.echo(diff) + else: + click.echo("No diff to show.") print() return bool(len(check_file.edited_files)) From 52a339d2799b84c2edd61883ade1c75207fca332 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 23:46:50 -0500 Subject: [PATCH 51/99] nit: use `dictionary` instead of `dit` to not look like a typo --- modmail/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 7a0f24b4..35a8eea4 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -557,15 +557,15 @@ def load_yaml(path: os.PathLike) -> dict: DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) -def _remove_extra_values(klass: type, dit: DictT) -> DictT: +def _remove_extra_values(klass: type, dictionary: DictT) -> DictT: """ Remove extra values from the provided dict which don't fit into the provided klass recursively. klass must be an attr.s class. """ fields = attr.fields_dict(klass) - cleared_dict = dit.copy() - for k in dit: + cleared_dict = dictionary.copy() + for k in dictionary: if k not in fields: del cleared_dict[k] elif isinstance(cleared_dict[k], dict): From 0937a13239b095d424abd3445ccb382223d69847 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 00:06:35 -0500 Subject: [PATCH 52/99] config: rename .env.template to template.env --- .gitignore | 1 + .pre-commit-config.yaml | 2 +- docs/contributing.md | 2 +- scripts/export_new_config_to_default_config.py | 12 ++++++++---- .env.template => template.env | 0 5 files changed, 11 insertions(+), 6 deletions(-) rename .env.template => template.env (100%) diff --git a/.gitignore b/.gitignore index d52be37a..f3bddfd2 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,7 @@ celerybeat.pid # Environments .env +!template.env .venv env/ venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41e0d9ec..a4fe9d2b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: name: Export default configuration language: python entry: poetry run python -m scripts.export_new_config_to_default_config - files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + files: '(app\.json|template\.env|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' require_serial: true additional_dependencies: # so apparently these are needed, but the versions don't have to be pinned since it uses the local env diff --git a/docs/contributing.md b/docs/contributing.md index 4f351a13..5ade7573 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -181,7 +181,7 @@ $ poetry install 2. Set the modmail bot prefix in `bot.prefix`. 3. In case you are a contributor set `dev.mode.plugin_dev` and `dev.mode.develop` to `true`. The `develop` variable enables the developer bot extensions and `plugin_dev` enables plugin-developer friendly bot extensions. 4. Create a text file named `.env` in your project root (that's the base folder of your repository): - - You can also copy the `.env.template` file to `.env` + - You can also copy the `template.env` file to `.env` !!!note The entire file name is literally `.env` diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 1190000a..620b0e4c 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -32,7 +32,7 @@ MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent -ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" +ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / "template.env" APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" METADATA_TABLE = modmail.config.METADATA_TABLE @@ -97,9 +97,13 @@ def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 diff = difflib.unified_diff( original_contents, new_contents, fromfile="before", tofile="after" ) - diff = "".join(diff) - if diff is not None and pygments is not None: - diff = pygments.highlight(diff, DiffLexer(), Terminal256Formatter()) + try: + diff = "".join(diff) + except TypeError: + diff = None + else: + if pygments is not None: + diff = pygments.highlight(diff, DiffLexer(), Terminal256Formatter()) self.edited_files[file] = diff diff --git a/.env.template b/template.env similarity index 100% rename from .env.template rename to template.env From 3c60d4c4c76c7c11825e1691c782a07ec0fcc8d8 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 00:17:06 -0500 Subject: [PATCH 53/99] config(docs): document configuration loading priority --- docs/contributing.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/contributing.md b/docs/contributing.md index 5ade7573..94f46f06 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -188,6 +188,19 @@ $ poetry install 5. Open the file with any text editor and write the bot token to the files in this format: `MODMAIL_BOT_TOKEN="my_token"`. + +Of the several supported configuration sources, they are loaded in a specific priority. In decreasing priority: +- `os.environ` +- .env +- modmail_config.yaml (if PyYaml is installed and the file exists) +- modmail_config.toml (if the above file does not exist) +- defaults + +Internally, the actual parsing order may not match the above, +but the end configuration object will have the above priorty. + + + ### Run The Project To run the project, use the (below) in the project root. From 5ad572d7c76569780e62c5d4c5e260b0a6694151 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 00:45:03 -0500 Subject: [PATCH 54/99] minor: touch up comments and flatten branches --- modmail/config.py | 51 ++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 35a8eea4..ad60691c 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -67,10 +67,7 @@ class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter) """ async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: - # match = self._get_id_match(argument) or re.match( - # r"$", argument - # ) - + """Convert a provided argument into an emoji object.""" match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) if match is not None: groups = match.groupdict() @@ -196,8 +193,7 @@ class ConfigMetadata: types.FunctionType ] = None # if we want an attribute off of the converted value - # hidden, eg log_level - # hidden values mean they do not show up in the bot configuration menu + # hidden values do not show up in the bot configuration menu hidden: bool = False @description.validator @@ -447,13 +443,15 @@ def _build_class( if env_prefix is None: env_prefix = ENV_PREFIX - if env is None: - if dotenv_file is not None: - env = dotenv.dotenv_values(dotenv_file) - env.update(os.environ) - else: - env = os.environ.copy() - + # while dotenv has methods to load the .env into the environment, we don't want that + # that would mean that all of the .env variables would be loaded in to the env + # we can't assume everything in .env is for modmail + # so we load it into our own dictonary and parse from that + env = {} + if dotenv_file is not None: + env.update(dotenv.dotenv_values(dotenv_file)) + env.update(os.environ.copy()) + dotenv.load_dotenv() # get the attributes of the provided class if defaults is None: defaults = defaultdict(lambda: None) @@ -464,22 +462,29 @@ def _build_class( for var in attr.fields(klass): if attr.has(var.type): - # var is an attrs class too + # var is an attrs class too, so recurse over it kw[var.name] = _build_class( var.type, env=env, env_prefix=env_prefix + var.name.upper() + "_", defaults=defaults.get(var.name, None), ) - else: - kw[var.name] = env.get(env_prefix + var.name.upper(), None) - if kw[var.name] is None: - if (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing: - kw[var.name] = defaults[var.name] - elif var.default is not attr.NOTHING: # check for var default - kw[var.name] = var.default - else: - del kw[var.name] + continue + + kw[var.name] = env.get(env_prefix + var.name.upper(), None) + + if kw[var.name] is not None: + continue + + if var.name in defaults and defaults[var.name] not in [None, marshmallow.missing]: + kw[var.name] = defaults[var.name] + continue + + if var.default is not attr.NOTHING: # check for var default + kw[var.name] = var.default + continue + + del kw[var.name] return klass(**kw) From 8bc7c2771885ba09b6fff115d68dcbb2274aa063 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 20:07:19 -0500 Subject: [PATCH 55/99] fix: use commands.run_converters() in the configuration manager ensures that we use the cache and special conversion methods to convert variables this means that all of the varaible conversion is handled by discord.py all conversion errors are propagated outwards and caught by our error handler --- modmail/extensions/configuration_manager.py | 55 ++++++++++----------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 076d2596..7b212bb0 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,3 +1,4 @@ +import inspect import logging import operator import string @@ -10,7 +11,6 @@ import marshmallow from discord.ext import commands from discord.ext.commands import Context -from discord.ext.commands import converter as commands_converter from modmail import config from modmail.bot import ModmailBot @@ -128,7 +128,8 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: return new_arg else: raise commands.BadArgument( - f"{ctx.current_parameter.name.capitalize()} `{arg}` is not a valid configuration key." + f"{ctx.current_parameter.name.capitalize()} `{arg}` is not a valid configuration key.\n" + f"{ctx.current_parameter.name.capitalize()} must be in {', '.join(fields.keys())}" ) @@ -180,46 +181,42 @@ async def list_config(self, ctx: Context) -> None: @config_group.command(name="set", aliases=("edit",)) async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" - if option not in self.config_fields: - raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") - meta = self.config_fields[option] + metadata = self.config_fields[option] - if meta.frozen: - - await responses.send_negatory_response(ctx, f"Unable to modify {option}.") + if metadata.frozen: + await responses.send_negatory_response( + ctx, f"Unable to modify `{option}` as it is frozen and cannot be edited during runtime." + ) return - if meta.modmail_metadata.discord_converter is not None: - value = await meta.modmail_metadata.discord_converter().convert(ctx, value) - if meta.modmail_metadata.discord_converter_attribute is not None: - if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): - value = meta.modmail_metadata.discord_converter_attribute(value) - if meta._type in commands_converter.CONVERTER_MAPPING: - value = await commands_converter.CONVERTER_MAPPING[meta._type]().convert(ctx, value) - if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): - value = meta.modmail_metadata.discord_converter_attribute(value) - elif meta._field.converter: - value = meta._field.converter(value) + # since we are in the bot, we are able to run commands.run_converters() + # we have a context object, and several other objects, so this should be able to work + param = inspect.Parameter("value", 1, default=metadata.default, annotation=metadata._type) + converted_result = await commands.run_converters(ctx, metadata._type, value, param) + + discord_converter_attribute = metadata.modmail_metadata.discord_converter_attribute + if isinstance(discord_converter_attribute, types.FunctionType): + converted_result = discord_converter_attribute(converted_result) if "." in option: - root, name = option.rsplit(".", -1) + root, name = option.rsplit(".", 1) else: root = "" name = option - setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + setattr(operator.attrgetter(root)(self.bot.config.user), name, converted_result) - await responses.send_positive_response(ctx, f"Successfully set `{option}` to `{value}`.") + if converted_result == value: + response = f"Successfully set `{option}` to `{value}`." + else: + response = f"Successfully set `{option}` to `{converted_result}` (converted from `{value}`)" + await responses.send_positive_response(ctx, response) @config_group.command(name="get", aliases=("show",)) async def get_config(self, ctx: Context, option: KeyConverter) -> None: - """Modify an existing configuration value.""" - if option not in self.config_fields: - raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") - - get_value = operator.attrgetter(option) - value = get_value(self.bot.config.user) - await ctx.send(f"{option}: `{value}`") + """Display an existing configuration value.""" + value = operator.attrgetter(option)(self.bot.config.user) + await responses.send_general_response(ctx, f"value: `{value}`", embed=discord.Embed(title=option)) def setup(bot: ModmailBot) -> None: From 263ad6bd50d7e824ddd02a7ac010c0a1b695379e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 20:10:30 -0500 Subject: [PATCH 56/99] config-manager: add set-default command to reset to default --- modmail/extensions/configuration_manager.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 7b212bb0..5bb7e57d 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -178,6 +178,31 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message, embed=embed) + @config_group.command(name="set_default", aliases=("set-default",)) + async def set_default(self, ctx: Context, option: KeyConverter) -> None: + """Reset the provided configuration value to the default.""" + if "." in option: + root, name = option.rsplit(".", 1) + else: + root = "" + name = option + value = operator.attrgetter(option)(self.bot.config.default) + if value in [marshmallow.missing, attr.NOTHING]: + await responses.send_negatory_response( + ctx, f"`{option}` is a required configuration variable and cannot be reset." + ) + return + try: + setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + except (attr.exceptions.FrozenAttributeError, attr.exceptions.FrozenInstanceError): + await responses.send_negatory_response( + ctx, f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." + ) + else: + await responses.send_positive_response( + ctx, f"Successfully set `{option}` to the default of `{value}`." + ) + @config_group.command(name="set", aliases=("edit",)) async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" From 0a596b4d2f55372d9aa21e3c4aefb64f08576366 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 20:39:34 -0500 Subject: [PATCH 57/99] add set default command and offload setting config to a helper method --- modmail/extensions/configuration_manager.py | 60 ++++++++++++--------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 5bb7e57d..21933400 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -27,6 +27,12 @@ KeyT = str +class UnableToModifyConfig(commands.CommandError): + """Raised when a command is unable to modify the configuration.""" + + pass + + @attr.mutable class ConfOptions: """Configuration attribute class.""" @@ -178,33 +184,45 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message, embed=embed) - @config_group.command(name="set_default", aliases=("set-default",)) - async def set_default(self, ctx: Context, option: KeyConverter) -> None: - """Reset the provided configuration value to the default.""" + _T = typing.TypeVar("_T") + + async def set_config_value(self, option: str, new_value: _T) -> typing.Tuple[str, _T]: + """ + Set the provided option to new_value. + + Raises an UnableToModifyConfig error if there is an issue + """ if "." in option: root, name = option.rsplit(".", 1) else: root = "" name = option - value = operator.attrgetter(option)(self.bot.config.default) - if value in [marshmallow.missing, attr.NOTHING]: - await responses.send_negatory_response( - ctx, f"`{option}` is a required configuration variable and cannot be reset." - ) - return + + if new_value in [marshmallow.missing, attr.NOTHING]: + raise UnableToModifyConfig( + f"`{option}` is a required configuration variable and cannot be reset." + ) from None try: - setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + setattr(operator.attrgetter(root)(self.bot.config.user), name, new_value) except (attr.exceptions.FrozenAttributeError, attr.exceptions.FrozenInstanceError): - await responses.send_negatory_response( - ctx, f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." - ) + raise UnableToModifyConfig( + f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." + ) from None else: - await responses.send_positive_response( - ctx, f"Successfully set `{option}` to the default of `{value}`." - ) + return (option, new_value) + + @config_group.command(name="set_default", aliases=("set-default",)) + async def set_default(self, ctx: Context, option: KeyConverter) -> None: + """Reset the provided configuration value to the default.""" + value = operator.attrgetter(option)(self.bot.config.default) + await self.set_config_value(option, value) + + await responses.send_positive_response( + ctx, f"Successfully set `{option}` to the default of `{value}`." + ) @config_group.command(name="set", aliases=("edit",)) - async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: + async def modify_config_command(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" metadata = self.config_fields[option] @@ -223,13 +241,7 @@ async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> if isinstance(discord_converter_attribute, types.FunctionType): converted_result = discord_converter_attribute(converted_result) - if "." in option: - root, name = option.rsplit(".", 1) - else: - root = "" - name = option - - setattr(operator.attrgetter(root)(self.bot.config.user), name, converted_result) + await self.set_config_value(option, value) if converted_result == value: response = f"Successfully set `{option}` to `{value}`." From 81c08da536125913aed657ff417dee10b91714c1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 21:00:54 -0500 Subject: [PATCH 58/99] fix: add missing character to filter, fix env loading --- modmail/config.py | 11 ++++------- modmail/extensions/configuration_manager.py | 2 +- modmail/utils/extensions.py | 3 ++- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index ad60691c..4a2146f8 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -428,7 +428,7 @@ class Config: def _build_class( klass: ClassT, env: typing.Dict[str, str] = None, - env_prefix: str = None, + env_prefix: str = ENV_PREFIX, *, dotenv_file: os.PathLike = None, defaults: typing.Dict = None, @@ -440,9 +440,6 @@ def _build_class( Also can parse from a provided dictionary of environment variables. If `defaults` is provided, uses a value from there if the environment variable is not set or is None. """ - if env_prefix is None: - env_prefix = ENV_PREFIX - # while dotenv has methods to load the .env into the environment, we don't want that # that would mean that all of the .env variables would be loaded in to the env # we can't assume everything in .env is for modmail @@ -451,7 +448,7 @@ def _build_class( if dotenv_file is not None: env.update(dotenv.dotenv_values(dotenv_file)) env.update(os.environ.copy()) - dotenv.load_dotenv() + # get the attributes of the provided class if defaults is None: defaults = defaultdict(lambda: None) @@ -503,9 +500,9 @@ def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> di if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) + new_config_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) - return existing_cfg_dict + return new_config_dict def load_toml(path: os.PathLike = None) -> defaultdict: diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 21933400..002cfeeb 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -125,7 +125,7 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: new_arg = "" for c in arg.lower(): - if c in " /`": + if c in "./-": new_arg += "." else: new_arg += c diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index b7bf7cae..5e5d47fd 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -34,8 +34,9 @@ def determine_bot_mode() -> int: The configuration system uses true/false values, so we need to turn them into an integer for bitwise. """ bot_mode = 0 + _config = config() for mode in BotModes: - if getattr(config().user.dev.mode, unqualify(str(mode)).lower(), True): + if getattr(_config.user.dev.mode, unqualify(str(mode)).lower(), True): bot_mode += mode.value return bot_mode From 196d2a8a640e342c9530b661827271007c4d16cf Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 01:17:37 -0500 Subject: [PATCH 59/99] fix: use metadata provided converter to convert objects --- modmail/config.py | 4 ++-- modmail/extensions/configuration_manager.py | 24 +++++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 4a2146f8..654b8274 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -348,7 +348,7 @@ class EmojiCfg: This was a pain to implement. """ - success: typing.Any = attr.ib( + success: str = attr.ib( default=":thumbsup:", metadata={ METADATA_TABLE: ConfigMetadata( @@ -359,7 +359,7 @@ class EmojiCfg: }, ) - failure: typing.Any = attr.ib( + failure: str = attr.ib( default=":x:", metadata={ METADATA_TABLE: ConfigMetadata( diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 002cfeeb..9c2c0a2e 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -44,11 +44,13 @@ class ConfOptions: extended_description: str hidden: bool - _type: type + type: type metadata: dict modmail_metadata: config.ConfigMetadata + discord_converter: commands.Converter + discord_converter_attribute: types.FunctionType = None _field: attr.Attribute = None nested: str = None @@ -67,12 +69,15 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str kw["frozen"] = field.on_setattr is attr.setters.frozen or frozen - meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] - kw[config.METADATA_TABLE] = meta - kw["description"] = meta.description - kw["canonical_name"] = meta.canonical_name - kw["extended_description"] = meta.extended_description - kw["hidden"] = meta.hidden + metadata_table: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] + kw[config.METADATA_TABLE] = metadata_table + kw["description"] = metadata_table.description + kw["canonical_name"] = metadata_table.canonical_name + kw["extended_description"] = metadata_table.extended_description + kw["hidden"] = metadata_table.hidden + + kw["discord_converter"] = metadata_table.discord_converter + kw["discord_converter_attribute"] = metadata_table.discord_converter_attribute if nested is not None: kw["nested"] = nested @@ -234,8 +239,9 @@ async def modify_config_command(self, ctx: Context, option: KeyConverter, value: # since we are in the bot, we are able to run commands.run_converters() # we have a context object, and several other objects, so this should be able to work - param = inspect.Parameter("value", 1, default=metadata.default, annotation=metadata._type) - converted_result = await commands.run_converters(ctx, metadata._type, value, param) + annotation = metadata.discord_converter or metadata.type + param = inspect.Parameter("value", 1, default=metadata.default, annotation=annotation) + converted_result = await commands.run_converters(ctx, annotation, value, param) discord_converter_attribute = metadata.modmail_metadata.discord_converter_attribute if isinstance(discord_converter_attribute, types.FunctionType): From ebcaf9485e416619e22f5b12a2e05e4cc0e23ee9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 01:59:27 -0500 Subject: [PATCH 60/99] nits: standardize naming and make class names more self explanatory cfg, config, configuration... why so many similarily named classes??? --- modmail/config.py | 89 +++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 654b8274..3b894ba5 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -31,15 +31,14 @@ "ENV_PREFIX", "USER_CONFIG_FILE_NAME", "USER_CONFIG_FILES", - "CfgLoadError", + "ConfigLoadError", "Config", "config", "ConfigurationSchema", - "BotCfg", - "BotModeCfg", - "Cfg", - "Colours", - "DevCfg", + "BotConfig", + "BotModeConfig", + "ColourConfig", + "DeveloperConfig", "convert_to_color", "get_config", "get_default_config", @@ -80,7 +79,7 @@ async def convert(self, _: discord.ext.commands.context.Context, argument: str) # load env before we do *anything* -# TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. +# !!! TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. # TODO: add the above feature to the other configuration locations too. dotenv.load_dotenv(_CWD / ".env") @@ -90,10 +89,10 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -class CfgLoadError(Exception): +class ConfigLoadError(Exception): """Exception if the configuration failed to load from a local file.""" - ... + pass class _ColourField(marshmallow.fields.Field): @@ -157,7 +156,7 @@ def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Col @attr.frozen(kw_only=True) class ConfigMetadata: """ - Cfg metadata. This is intended to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. + Config metadata. The intent is to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. Nearly all of these values are optional, save for the description. All of them are keyword only. @@ -210,7 +209,7 @@ def _validate_discord_converter(self, attrib: attr.Attribute, value: typing.Any) @attr.s(auto_attribs=True, slots=True) -class BotCfg: +class BotConfig: """ Values that are configuration for the bot itself. @@ -257,7 +256,7 @@ class BotCfg: @attr.s(auto_attribs=True, slots=True, frozen=True) -class BotModeCfg: +class BotModeConfig: """ The three bot modes for the bot. Enabling some of these may enable other bot features. @@ -299,7 +298,7 @@ class BotModeCfg: @attr.s(auto_attribs=True, slots=True) -class Colours: +class ColourConfig: """ Default colors. @@ -319,10 +318,10 @@ class Colours: @attr.s(auto_attribs=True, slots=True) -class DevCfg: +class DeveloperConfig: """Developer configuration. These values should not be changed unless you know what you're doing.""" - mode: BotModeCfg = BotModeCfg() + mode: BotModeConfig = BotModeConfig() log_level: int = attr.ib( default=logging.INFO, metadata={ @@ -341,7 +340,7 @@ def _log_level_validator(self, a: attr.Attribute, value: int) -> None: @attr.mutable(slots=True) -class EmojiCfg: +class EmojiConfig: """ Emojis used across the entire bot. @@ -372,7 +371,7 @@ class EmojiCfg: @attr.s(auto_attribs=True, slots=True) -class Cfg: +class BaseConfig: """ Base configuration attrs class. @@ -380,10 +379,10 @@ class Cfg: we can get a clean default variable if we don't pass anything. """ - bot: BotCfg = BotCfg() - colours: Colours = Colours() - dev: DevCfg = attr.ib( - default=DevCfg(), + bot: BotConfig = BotConfig() + colours: ColourConfig = ColourConfig() + dev: DeveloperConfig = attr.ib( + default=DeveloperConfig(), metadata={ METADATA_TABLE: ConfigMetadata( description=( @@ -393,15 +392,15 @@ class Cfg: ) }, ) - emojis: EmojiCfg = EmojiCfg() + emojis: EmojiConfig = EmojiConfig() # build configuration -ConfigurationSchema = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 +ConfigurationSchema = desert.schema_class(BaseConfig, meta={"ordered": True}) # noqa: N818 _CACHED_CONFIG: "Config" = None -_CACHED_DEFAULT: Cfg = None +_CACHED_DEFAULT: BaseConfig = None @attr.s(auto_attribs=True, slots=True, kw_only=True) @@ -410,15 +409,15 @@ class Config: Base configuration variable. Used across the entire bot for configuration variables. Holds two variables, default and user. - Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + Default is a BaseConfig instance with nothing passed. It is a default instance of BaseConfig. - User is a Cfg schema instance, generated from a combination of the defaults, + User is a BaseConfig schema instance, generated from a combination of the defaults, user provided toml, and environment variables. """ - user: Cfg + user: BaseConfig schema: marshmallow.Schema - default: Cfg = Cfg() + default: BaseConfig = BaseConfig() ClassT = typing.TypeVar("ClassT", bound=type) @@ -486,7 +485,7 @@ def _build_class( return klass(**kw) -def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: +def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> dict: """ Load a configuration dictionary from the specified env file and environment variables. @@ -497,10 +496,16 @@ def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> di else: env_file = pathlib.Path(env_file) - if not existing_cfg_dict: - existing_cfg_dict = defaultdict(_generate_default_dict) + if not existing_config_dict: + existing_config_dict = defaultdict(_generate_default_dict) - new_config_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) + new_config_dict = attr.asdict( + _build_class( + BaseConfig, + dotenv_file=env_file, + defaults=existing_config_dict, + ) + ) return new_config_dict @@ -518,13 +523,13 @@ def load_toml(path: os.PathLike = None) -> defaultdict: path = pathlib.Path(path) if not path.is_file(): - raise CfgLoadError("The provided toml file path is not a valid file.") + raise ConfigLoadError("The provided toml file path is not a valid file.") try: with open(path) as f: return defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) except Exception as e: - raise CfgLoadError from e + raise ConfigLoadError from e def load_yaml(path: os.PathLike) -> dict: @@ -547,13 +552,13 @@ def load_yaml(path: os.PathLike) -> dict: ("The provided yaml config file is not a readable file.", path.is_file()), ] if errors := "\n".join(msg for msg, check in states if not check): - raise CfgLoadError(errors) + raise ConfigLoadError(errors) try: with open(path, "r") as f: return defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) except Exception as e: - raise CfgLoadError from e + raise ConfigLoadError from e DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) @@ -590,7 +595,7 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: """ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn: - raise CfgLoadError( + raise ConfigLoadError( f"The required dependency for reading {file_type} configuration files is not installed. " f"Please install {dependency or file_type} to allow reading these files." ) @@ -617,16 +622,16 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn loaded_config_dict = load_yaml(file) break else: - raise CfgLoadError("Provided configuration file is not of a supported type.") + raise ConfigLoadError("Provided configuration file is not of a supported type.") if should_load_env: - loaded_config_dict = load_env(existing_cfg_dict=loaded_config_dict) + loaded_config_dict = load_env(existing_config_dict=loaded_config_dict) # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. - loaded_config_dict = _remove_extra_values(Cfg, loaded_config_dict) + loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) @@ -644,11 +649,11 @@ def get_config() -> Config: return _CACHED_CONFIG -def get_default_config() -> Cfg: +def get_default_config() -> BaseConfig: """Get the default configuration instance of the global Config instance.""" global _CACHED_DEFAULT if _CACHED_DEFAULT is None: - _CACHED_DEFAULT = Cfg() + _CACHED_DEFAULT = BaseConfig() return _CACHED_DEFAULT From 2fd074c705f42b27acd9c0ef625656a0a566b989 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 09:18:29 -0500 Subject: [PATCH 61/99] config: make dotenv optional and better errors when missing variables --- modmail/config.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 3b894ba5..8a8dff96 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -11,12 +11,16 @@ import discord import discord.ext.commands import discord.ext.commands.converter -import dotenv import marshmallow import marshmallow.fields import marshmallow.validate +try: + import dotenv +except ModuleNotFoundError: # pragma: nocover + dotenv = None + try: import atoml except ModuleNotFoundError: # pragma: nocover @@ -57,6 +61,8 @@ _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] +logger = logging.getLogger(__name__) + class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): """ @@ -78,12 +84,6 @@ async def convert(self, _: discord.ext.commands.context.Context, argument: str) return discord.PartialEmoji(name=argument) -# load env before we do *anything* -# !!! TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. -# TODO: add the above feature to the other configuration locations too. -dotenv.load_dotenv(_CWD / ".env") - - def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" return defaultdict(_generate_default_dict) @@ -443,8 +443,12 @@ def _build_class( # that would mean that all of the .env variables would be loaded in to the env # we can't assume everything in .env is for modmail # so we load it into our own dictonary and parse from that - env = {} + if env is None: + env = {} + if dotenv_file is not None: + if dotenv is None: + raise ConfigLoadError("dotenv extra must be installed to read .env files.") env.update(dotenv.dotenv_values(dotenv_file)) env.update(os.environ.copy()) @@ -492,7 +496,9 @@ def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> All dependencies for this will always be installed. """ if env_file is None: - env_file = _CWD / ".env" + # only presume an .env file if dotenv is not None + if dotenv is not None: + env_file = _CWD / ".env" else: env_file = pathlib.Path(env_file) @@ -627,13 +633,16 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn if should_load_env: loaded_config_dict = load_env(existing_config_dict=loaded_config_dict) - # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them + # HACK remove extra keys from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) - - loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) + try: + loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) + except marshmallow.ValidationError: + logger.exception("Unable to load the configuration.") + exit(1) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From fa02b3436bee93b34211930d830fb7b9069d31c1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 11:11:21 -0500 Subject: [PATCH 62/99] config: add logging and stabilize configuration directory --- modmail/config.py | 82 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 8a8dff96..faf5e36c 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,3 +1,4 @@ +import functools import inspect import logging import os @@ -15,23 +16,34 @@ import marshmallow.fields import marshmallow.validate +from modmail.log import ModmailLogger + +_log_optional_deps = logging.getLogger("modmail.optional_dependencies") +_silence = " You can silence these alerts by ignoring the `modmail.optional_dependencies` logger" try: import dotenv except ModuleNotFoundError: # pragma: nocover + _log_optional_deps.notice("dotenv was unable to be imported." + _silence) dotenv = None try: import atoml except ModuleNotFoundError: # pragma: nocover + _log_optional_deps.notice("atoml was unable to be imported." + _silence) atoml = None try: import yaml except ModuleNotFoundError: # pragma: nocover + _log_optional_deps.notice("yaml was unable to be imported." + _silence) yaml = None +del _log_optional_deps +del _silence + __all__ = [ "AUTO_GEN_FILE_NAME", + "CONFIG_DIRECTORY", "ENV_PREFIX", "USER_CONFIG_FILE_NAME", "USER_CONFIG_FILES", @@ -51,18 +63,45 @@ "load_yaml", ] -_CWD = pathlib.Path.cwd() + +logger: ModmailLogger = logging.getLogger(__name__) + + +@functools.lru_cache +def _get_config_directory() -> pathlib.Path: + """ + Return a directory where the configuration should reside. + + This is typically the current working directory, with a cavet. + + If the current directory's path includes the first parent directory of the bot + then the parent directory of the bot is looked for configuration files. + """ + import modmail + + path = pathlib.Path(modmail.__file__).parent.parent + cwd = pathlib.Path.cwd() + try: + cwd.relative_to(path) + except ValueError: + result = cwd + else: + result = path + logger.trace(f"Determined bot root directory from cwd as {result}") + logger.trace("That directory is either the cwd, or the common parent of the bot and cwd directories.") + return result + + +CONFIG_DIRECTORY = _get_config_directory() METADATA_TABLE = "modmail_metadata" ENV_PREFIX = "MODMAIL_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" USER_CONFIG_FILES = [ - _CWD / (USER_CONFIG_FILE_NAME + ".yaml"), - _CWD / (USER_CONFIG_FILE_NAME + ".toml"), + CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml"), + CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml"), ] -logger = logging.getLogger(__name__) - class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): """ @@ -431,6 +470,7 @@ def _build_class( *, dotenv_file: os.PathLike = None, defaults: typing.Dict = None, + skip_dotenv_if_none: bool = False, ) -> ClassT: """ Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. @@ -447,9 +487,12 @@ def _build_class( env = {} if dotenv_file is not None: - if dotenv is None: + if dotenv is not None: + env.update(dotenv.dotenv_values(dotenv_file)) + elif not skip_dotenv_if_none: raise ConfigLoadError("dotenv extra must be installed to read .env files.") - env.update(dotenv.dotenv_values(dotenv_file)) + else: + logger.info(f"{dotenv = }, but {skip_dotenv_if_none = }" f" so ignoring reading {dotenv_file = }") env.update(os.environ.copy()) # get the attributes of the provided class @@ -489,16 +532,18 @@ def _build_class( return klass(**kw) -def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> dict: +def load_env( + env_file: os.PathLike = None, existing_config_dict: dict = None, *, skip_dotenv_if_none: bool = False +) -> dict: """ Load a configuration dictionary from the specified env file and environment variables. All dependencies for this will always be installed. """ + logger.trace(f"function `load_env` called to load env_file {env_file}") + if env_file is None: - # only presume an .env file if dotenv is not None - if dotenv is not None: - env_file = _CWD / ".env" + env_file = CONFIG_DIRECTORY / ".env" else: env_file = pathlib.Path(env_file) @@ -510,6 +555,7 @@ def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> BaseConfig, dotenv_file=env_file, defaults=existing_config_dict, + skip_dotenv_if_none=skip_dotenv_if_none, ) ) @@ -522,8 +568,10 @@ def load_toml(path: os.PathLike = None) -> defaultdict: All dependencies for this will always be installed. """ + logger.trace(f"function `load_toml` called to load toml file {path}") + if path is None: - path = (_CWD / (USER_CONFIG_FILE_NAME + ".toml"),) + path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml"),) else: # fully resolve path path = pathlib.Path(path) @@ -547,8 +595,9 @@ def load_yaml(path: os.PathLike) -> dict: In order to keep errors at a minimum, this function checks if both pyyaml is installed and if a yaml configuration file exists before raising an exception. """ + logger.trace(f"function `load_yaml` called to load yaml file {path}") if path is None: - path = (_CWD / (USER_CONFIG_FILE_NAME + ".yaml"),) + path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml"),) else: path = pathlib.Path(path) @@ -601,10 +650,12 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: """ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn: - raise ConfigLoadError( + msg = ( f"The required dependency for reading {file_type} configuration files is not installed. " f"Please install {dependency or file_type} to allow reading these files." ) + logger.error(msg) + raise ConfigLoadError(msg) if len(files) == 0: files = USER_CONFIG_FILES @@ -631,13 +682,14 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn raise ConfigLoadError("Provided configuration file is not of a supported type.") if should_load_env: - loaded_config_dict = load_env(existing_config_dict=loaded_config_dict) + loaded_config_dict = load_env(existing_config_dict=loaded_config_dict, skip_dotenv_if_none=True) # HACK remove extra keys from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) + logger.debug("configuration loaded_config_dict prepped. Attempting to seralize...") try: loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) except marshmallow.ValidationError: From 3d1986d82bdb1d21b5965639f335fbe08631bd22 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 11:18:48 -0500 Subject: [PATCH 63/99] minor: use functools cache to cache the config --- modmail/config.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index faf5e36c..5f7b5e2d 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -438,10 +438,6 @@ class BaseConfig: ConfigurationSchema = desert.schema_class(BaseConfig, meta={"ordered": True}) # noqa: N818 -_CACHED_CONFIG: "Config" = None -_CACHED_DEFAULT: BaseConfig = None - - @attr.s(auto_attribs=True, slots=True, kw_only=True) class Config: """ @@ -698,24 +694,20 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) +@functools.lru_cache(None) def get_config() -> Config: """ Helps to try to ensure that only one instance of the Config class exists. This means that all usage of the configuration is using the same configuration class. """ - global _CACHED_CONFIG - if _CACHED_CONFIG is None: - _CACHED_CONFIG = _load_config() - return _CACHED_CONFIG + return _load_config() +@functools.lru_cache(None) def get_default_config() -> BaseConfig: """Get the default configuration instance of the global Config instance.""" - global _CACHED_DEFAULT - if _CACHED_DEFAULT is None: - _CACHED_DEFAULT = BaseConfig() - return _CACHED_DEFAULT + return BaseConfig() config = get_config From ff04a5ae2c6d8f73a231391e6c9fa848d2f9502e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 12:31:01 -0500 Subject: [PATCH 64/99] use factories to generate mutable classes and add more logging tests: mark xfail the test for default config caching --- modmail/config.py | 41 +++++++++++++++++++++++------------- tests/modmail/test_config.py | 7 ++++-- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 5f7b5e2d..1050f007 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -360,7 +360,7 @@ class ColourConfig: class DeveloperConfig: """Developer configuration. These values should not be changed unless you know what you're doing.""" - mode: BotModeConfig = BotModeConfig() + mode: BotModeConfig = attr.ib(factory=BotModeConfig) log_level: int = attr.ib( default=logging.INFO, metadata={ @@ -418,10 +418,10 @@ class BaseConfig: we can get a clean default variable if we don't pass anything. """ - bot: BotConfig = BotConfig() - colours: ColourConfig = ColourConfig() + bot: BotConfig = attr.ib(factory=BotConfig) + colours: ColourConfig = attr.ib(factory=ColourConfig) dev: DeveloperConfig = attr.ib( - default=DeveloperConfig(), + factory=DeveloperConfig, metadata={ METADATA_TABLE: ConfigMetadata( description=( @@ -438,7 +438,17 @@ class BaseConfig: ConfigurationSchema = desert.schema_class(BaseConfig, meta={"ordered": True}) # noqa: N818 -@attr.s(auto_attribs=True, slots=True, kw_only=True) +# @functools.lru_cache(None) +def get_default_config() -> BaseConfig: + """Get the default configuration instance of the BaseConfig instance.""" + # TODO: This should be a frozen instance all the way down on every level + # TODO: but I'm not sure the best way to do that, or even how to do that + # TODO: as a result, the lrucache decorator is temporarily commented out + # TODO: until we discover the best way to freeze this and all subclasses + return BaseConfig() + + +@attr.s(auto_attribs=True, slots=True, kw_only=True, frozen=True) class Config: """ Base configuration variable. Used across the entire bot for configuration variables. @@ -452,7 +462,7 @@ class Config: user: BaseConfig schema: marshmallow.Schema - default: BaseConfig = BaseConfig() + default: BaseConfig = get_default_config() ClassT = typing.TypeVar("ClassT", bound=type) @@ -577,10 +587,13 @@ def load_toml(path: os.PathLike = None) -> defaultdict: try: with open(path) as f: - return defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) + result = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) except Exception as e: raise ConfigLoadError from e + logger.info(f"Successfully parsed toml config located at {path!s}") + return result + def load_yaml(path: os.PathLike) -> dict: """ @@ -607,10 +620,13 @@ def load_yaml(path: os.PathLike) -> dict: try: with open(path, "r") as f: - return defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) + result = defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) except Exception as e: raise ConfigLoadError from e + logger.info(f"Successfully parsed yaml config located at {path!s}") + return result + DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) @@ -685,12 +701,13 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) - logger.debug("configuration loaded_config_dict prepped. Attempting to seralize...") + logger.debug("Configuration loaded_config_dict prepped. Attempting to seralize...") try: loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) except marshmallow.ValidationError: logger.exception("Unable to load the configuration.") exit(1) + logger.debug("Seralization successful.") return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) @@ -704,11 +721,5 @@ def get_config() -> Config: return _load_config() -@functools.lru_cache(None) -def get_default_config() -> BaseConfig: - """Get the default configuration instance of the global Config instance.""" - return BaseConfig() - - config = get_config default = get_default_config diff --git a/tests/modmail/test_config.py b/tests/modmail/test_config.py index a7a580a8..4faad2e9 100644 --- a/tests/modmail/test_config.py +++ b/tests/modmail/test_config.py @@ -19,13 +19,16 @@ def test_config_is_cached(): """Test configuration is cached, helping keep only one version of the configuration in existance.""" for _ in range(2): - assert config.config() == config._CACHED_CONFIG + assert config.config() is config.config() +@pytest.mark.xfail( + reason="Default config is temporarily not cached due to a bug relating to frozen attributes." +) def test_default_config_is_cached(): """Test default configuration is cached, helping keep only one version of the config in existance.""" for _ in range(2): - assert config.default() == config._CACHED_DEFAULT + assert config.default() is config.default() class TestConfigLoaders: From 258fa1b39505aefa8e11e04ea89f16c240114aa5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 12:32:28 -0500 Subject: [PATCH 65/99] fix: use config.user to return configured prefix --- modmail/extensions/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/extensions/meta.py b/modmail/extensions/meta.py index 35139d6f..869a33eb 100644 --- a/modmail/extensions/meta.py +++ b/modmail/extensions/meta.py @@ -43,7 +43,7 @@ async def prefix(self, ctx: commands.Context) -> None: await ctx.send( embed=discord.Embed( title="Current Prefix", - description=f"My currently configured prefix is `{self.bot.config.bot.prefix}`.", + description=f"My currently configured prefix is `{self.bot.config.user.bot.prefix}`.", ) ) From bfe641c45f4645b1d8908086e29d436e2bcefd7b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 12:43:09 -0500 Subject: [PATCH 66/99] chore: export the extended description after the normal description to app.json --- app.json | 2 +- scripts/export_new_config_to_default_config.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app.json b/app.json index e0d64443..5fa7d2cf 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { "env": { "MODMAIL_BOT_TOKEN": { - "description": "Discord bot token. Required to log in to discord.", + "description": "Discord bot token. Required to log in to discord.\nThis is obtainable from https://discord.com/developers/applications", "required": true }, "MODMAIL_BOT_PREFIX": { diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 620b0e4c..854e1f00 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -229,11 +229,13 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada or meta[METADATA_TABLE].export_to_env_template or meta.get("required", False) ): - + description = ( + f"{meta[METADATA_TABLE].description}\n{meta[METADATA_TABLE].extended_description or ''}" + ).strip() options = defaultdict( str, { - "description": meta[METADATA_TABLE].description, + "description": description, "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), }, ) From fe53cf7b47d13f50df32ae561bf697a96c5d4bf3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 10:29:27 -0500 Subject: [PATCH 67/99] chore: remove operator.attrgetter and implement recursive setters and getters --- modmail/extensions/configuration_manager.py | 37 +++++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 9c2c0a2e..eee1b932 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,6 +1,5 @@ import inspect import logging -import operator import string import types import typing @@ -27,6 +26,27 @@ KeyT = str +def _recursive_getattr(obj: typing.Any, attribute: str) -> typing.Any: + """Get an attribute recursively. All `.` in attribute will be accessed recursively.""" + for name in attribute.split("."): + obj = getattr(obj, name) + return obj + + +def _recursive_setattr(obj: typing.Any, attribute: str, value: typing.Any) -> typing.Any: + """ + Get an attribute recursively. + + All `.` in attribute will be accessed recursively up to the final, which will be set. + """ + if "." in attribute: + root, attr = attribute.rsplit(".", 1) + obj = _recursive_getattr(obj, root) + + setattr(obj, attr, value) + return value + + class UnableToModifyConfig(commands.CommandError): """Raised when a command is unable to modify the configuration.""" @@ -197,18 +217,13 @@ async def set_config_value(self, option: str, new_value: _T) -> typing.Tuple[str Raises an UnableToModifyConfig error if there is an issue """ - if "." in option: - root, name = option.rsplit(".", 1) - else: - root = "" - name = option - if new_value in [marshmallow.missing, attr.NOTHING]: raise UnableToModifyConfig( - f"`{option}` is a required configuration variable and cannot be reset." + f"`{option.rsplit('.', 1)[-1]}` is a required configuration variable and cannot be reset." ) from None + try: - setattr(operator.attrgetter(root)(self.bot.config.user), name, new_value) + _recursive_setattr(self.bot.config.user, option, new_value) except (attr.exceptions.FrozenAttributeError, attr.exceptions.FrozenInstanceError): raise UnableToModifyConfig( f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." @@ -219,7 +234,7 @@ async def set_config_value(self, option: str, new_value: _T) -> typing.Tuple[str @config_group.command(name="set_default", aliases=("set-default",)) async def set_default(self, ctx: Context, option: KeyConverter) -> None: """Reset the provided configuration value to the default.""" - value = operator.attrgetter(option)(self.bot.config.default) + value = _recursive_getattr(self.bot.config.default, option) await self.set_config_value(option, value) await responses.send_positive_response( @@ -258,7 +273,7 @@ async def modify_config_command(self, ctx: Context, option: KeyConverter, value: @config_group.command(name="get", aliases=("show",)) async def get_config(self, ctx: Context, option: KeyConverter) -> None: """Display an existing configuration value.""" - value = operator.attrgetter(option)(self.bot.config.user) + value = _recursive_getattr(self.bot.config.user, option) await responses.send_general_response(ctx, f"value: `{value}`", embed=discord.Embed(title=option)) From 4e7fb22c0bdc79f33b7e52749ea726f60cd935d7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 10:50:24 -0500 Subject: [PATCH 68/99] chore: make the error spam less spammy for missing yaml file and dependency --- modmail/config.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 1050f007..c2a22325 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -610,12 +610,27 @@ def load_yaml(path: os.PathLike) -> dict: else: path = pathlib.Path(path) + # list of tuples containing a message, check, and stop checking. + # this is complicated in order to display the most errors at once to the user. + # this ensures that an attempt to load a yaml file is met with missing + # dependencies and non-existant file at the same time. states = [ - ("The yaml library is not installed.", yaml is not None), - ("The provided yaml config path does not exist.", path.exists()), - ("The provided yaml config file is not a readable file.", path.is_file()), + ("The yaml library is not installed.", yaml is not None, False), + ("The provided yaml config path does not exist.", path.exists(), True), + ("The provided yaml config file is not a regular file.", path.is_file(), False), ] - if errors := "\n".join(msg for msg, check in states if not check): + + errors = list() + for text, check, stop_checking in states: + if check: + continue + errors.append(text) + if stop_checking: + break + + if errors: + # formulate an error message + errors = "Errors occured while attempting to load a yaml configuration:\n" + "\n".join(errors) raise ConfigLoadError(errors) try: From 03fd57fd2a6e2168fbf8cebaa7bc1c1eebed3876 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 11:48:01 -0500 Subject: [PATCH 69/99] tests: fix environment not being replaced soon enough --- tests/modmail/conftest.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/modmail/conftest.py b/tests/modmail/conftest.py index 2e3b5139..ec3baa96 100644 --- a/tests/modmail/conftest.py +++ b/tests/modmail/conftest.py @@ -6,6 +6,9 @@ import pytest +_ORIG_ENVIRON = None + + def pytest_report_header(config) -> str: """Pytest headers.""" return "package: modmail" @@ -24,16 +27,16 @@ def _get_env(): def pytest_configure(): - """Check that the test specific env file exists, and cancel the run if it does not exist.""" + """Load the test specific environment file, exit if it does not exist.""" env = _get_env() if not env.is_file(): pytest.exit(f"Testing specific {env} does not exist. Cancelling test run.", 2) + os.environ.clear() + dotenv.load_dotenv(_get_env(), override=True) -@pytest.fixture(autouse=True, scope="package") -def standardize_environment(): - """Clear environment variables except for the test.env file.""" - env = _get_env() - with unittest.mock.patch.dict(os.environ, clear=True): - dotenv.load_dotenv(env) - yield +def pytest_unconfigure(): + """Reset os.environ to the original environment before the run.""" + if _ORIG_ENVIRON is not None: + os.environ.clear() + os.environ.update(**_ORIG_ENVIRON) From d0d7d7857f3faea41f513130a2b3342af86e5396 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 24 Nov 2021 10:07:15 -0500 Subject: [PATCH 70/99] ci: add coverage artifacts --- .github/workflows/lint_test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index 6db38e31..4c22f4c2 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -169,6 +169,15 @@ jobs: - name: Publish coverage report to codecov.io run: python -m codecov + - name: Coverage HTML + run: python -m coverage html + + - name: Upload coverage as artifact + uses: actions/upload-artifact@v2 + with: + name: coverage-${{ runner.os }}-python-${{ env.PYTHON_VERSION }} + path: htmlcov + artifact: name: Generate Artifact if: always() From dcf606b05fc3743bdc35a0f0b0b3d8bc245ef91e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 24 Nov 2021 10:52:34 -0500 Subject: [PATCH 71/99] ci(coverage): switch from codecov to coveralls --- .codecov.yml | 57 -------------- .github/workflows/lint_test.yml | 26 ++++++- README.md | 3 +- poetry.lock | 132 +++++++++++++++++++------------- pyproject.toml | 3 +- 5 files changed, 105 insertions(+), 116 deletions(-) delete mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 23325110..00000000 --- a/.codecov.yml +++ /dev/null @@ -1,57 +0,0 @@ -codecov: - require_ci_to_pass: yes - notify: - after_n_builds: 6 - wait_for_ci: yes - -coverage: - precision: 2 - round: down - range: "70...100" - status: - project: - default: false - bot: - paths: - - 'modmail/' - # basic - target: auto - threshold: 5% - base: auto - flags: - - unit - # advanced settings - branches: - - main - if_ci_failed: ignore #success, failure, error, ignore - informational: true - only_pulls: true - patch: - default: - # basic - target: auto - threshold: 5% - base: auto - # advanced - branches: - - main - if_ci_failed: ignore #success, failure, error, ignore - informational: true - only_pulls: true - - -parsers: - gcov: - branch_detection: - conditional: yes - loop: yes - method: no - macro: no - -comment: - layout: "reach,diff,flags,files,footer" - behavior: default - require_changes: no - -github_checks: - annotations: false diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index 4c22f4c2..7b520cc5 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -164,10 +164,15 @@ jobs: - name: Run tests and generate coverage report run: python -m pytest -n auto --dist loadfile --cov --disable-warnings -q - # This step will publish the coverage reports to codecov.io and + # This step will publish the coverage reports to coveralls.io and # print a "job" link in the output of the GitHub Action - - name: Publish coverage report to codecov.io - run: python -m codecov + - name: Publish coverage report to coveralls.io + env: + COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: coverage-${{ runner.os }}-python-${{ env.PYTHON_VERSION }} + COVERALLS_PARALLEL: true + COVERALLS_SERVICE_NAME: github + run: python -m coveralls - name: Coverage HTML run: python -m coverage html @@ -178,6 +183,21 @@ jobs: name: coverage-${{ runner.os }}-python-${{ env.PYTHON_VERSION }} path: htmlcov + + coveralls: + name: Indicate completion to coveralls.io + needs: test + runs-on: ubuntu-latest + container: python:3-slim + steps: + - name: Coveralls Finished + env: + COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_SERVICE_NAME: github + run: | + python3 -m pip install --upgrade coveralls + python3 -m coveralls --finish + artifact: name: Generate Artifact if: always() diff --git a/README.md b/README.md index 2e521aee..214746f2 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # Discord Modmail [![Lint & Test](https://img.shields.io/github/workflow/status/discord-modmail/modmail/Lint%20&%20Test/main?label=Lint+%26+Test&logo=github&style=flat)](https://github.com/discord-modmail/modmail/actions/workflows/lint_test.yml "Lint and Test") -[![Code Coverage](https://img.shields.io/codecov/c/gh/discord-modmail/modmail/main?logo=codecov&style=flat&label=Code+Coverage)](https://app.codecov.io/gh/discord-modmail/modmail "Code Coverage") -[![Codacy Grade](https://img.shields.io/codacy/grade/78be21a49835484595aea556d5920638?logo=codacy&style=flat&label=Code+Quality)](https://www.codacy.com/gh/discord-modmail/modmail/dashboard "Codacy Grade") +[![Code Coverage](https://img.shields.io/coveralls/github/discord-modmail/modmail?logo=coveralls&style=flat&label=Code+Coverage)](https://coveralls.io/github/discord-modmail/modmail) [![Python](https://img.shields.io/static/v1?label=Python&message=3.8+%7C+3.9&color=blue&logo=Python&style=flat)](https://www.python.org/downloads/ "Python 3.8 | 3.9") [![License](https://img.shields.io/github/license/discord-modmail/modmail?style=flat&label=License)](./LICENSE "License file") [![Code Style](https://img.shields.io/static/v1?label=Code%20Style&message=black&color=000000&style=flat)](https://github.com/psf/black "The uncompromising python formatter") diff --git a/poetry.lock b/poetry.lock index 71294634..8f4c1718 100644 --- a/poetry.lock +++ b/poetry.lock @@ -202,18 +202,6 @@ python-versions = ">=3.6" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "codecov" -version = "2.1.12" -description = "Hosted coverage reports for GitHub, Bitbucket and Gitlab" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -coverage = "*" -requests = ">=2.7.9" - [[package]] name = "colorama" version = "0.4.4" @@ -238,7 +226,7 @@ cron = ["capturer (>=2.4)"] [[package]] name = "coverage" -version = "6.0.2" +version = "6.1.2" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -250,6 +238,22 @@ tomli = {version = "*", optional = true, markers = "extra == \"toml\""} [package.extras] toml = ["tomli"] +[[package]] +name = "coveralls" +version = "3.3.1" +description = "Show coverage stats online via coveralls.io" +category = "dev" +optional = false +python-versions = ">= 3.5" + +[package.dependencies] +coverage = ">=4.1,<6.0.0 || >6.1,<6.1.1 || >6.1.1,<7.0" +docopt = ">=0.6.1" +requests = ">=1.0.0" + +[package.extras] +yaml = ["PyYAML (>=3.10)"] + [[package]] name = "discord.py" version = "2.0.0a0" @@ -278,6 +282,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "docopt" +version = "0.6.2" +description = "Pythonic argument parser, that will make you smile" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "execnet" version = "1.9.0" @@ -1206,7 +1218,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "b85bebf7a796b0b37e647fd7289589ea03a147c155839dfd1bcd561ead1fde92" +content-hash = "1086984557d7a4f7503e1f68d7a3a5662d2960998fec490144a3debdcc259256" [metadata.files] aiodns = [ @@ -1422,11 +1434,6 @@ click = [ {file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"}, {file = "click-8.0.3.tar.gz", hash = "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"}, ] -codecov = [ - {file = "codecov-2.1.12-py2.py3-none-any.whl", hash = "sha256:585dc217dc3d8185198ceb402f85d5cb5dbfa0c5f350a5abcdf9e347776a5b47"}, - {file = "codecov-2.1.12-py3.8.egg", hash = "sha256:782a8e5352f22593cbc5427a35320b99490eb24d9dcfa2155fd99d2b75cfb635"}, - {file = "codecov-2.1.12.tar.gz", hash = "sha256:a0da46bb5025426da895af90938def8ee12d37fcbcbbbc15b6dc64cf7ebc51c1"}, -] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, @@ -1436,45 +1443,66 @@ coloredlogs = [ {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, ] coverage = [ - {file = "coverage-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1549e1d08ce38259de2bc3e9a0d5f3642ff4a8f500ffc1b2df73fd621a6cdfc0"}, - {file = "coverage-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcae10fccb27ca2a5f456bf64d84110a5a74144be3136a5e598f9d9fb48c0caa"}, - {file = "coverage-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:53a294dc53cfb39c74758edaa6305193fb4258a30b1f6af24b360a6c8bd0ffa7"}, - {file = "coverage-6.0.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8251b37be1f2cd9c0e5ccd9ae0380909c24d2a5ed2162a41fcdbafaf59a85ebd"}, - {file = "coverage-6.0.2-cp310-cp310-win32.whl", hash = "sha256:db42baa892cba723326284490283a68d4de516bfb5aaba369b4e3b2787a778b7"}, - {file = "coverage-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:bbffde2a68398682623d9dd8c0ca3f46fda074709b26fcf08ae7a4c431a6ab2d"}, - {file = "coverage-6.0.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:60e51a3dd55540bec686d7fff61b05048ca31e804c1f32cbb44533e6372d9cc3"}, - {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a6a9409223a27d5ef3cca57dd7cd4dfcb64aadf2fad5c3b787830ac9223e01a"}, - {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4b34ae4f51bbfa5f96b758b55a163d502be3dcb24f505d0227858c2b3f94f5b9"}, - {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3bbda1b550e70fa6ac40533d3f23acd4f4e9cb4e6e77251ce77fdf41b3309fb2"}, - {file = "coverage-6.0.2-cp36-cp36m-win32.whl", hash = "sha256:4e28d2a195c533b58fc94a12826f4431726d8eb029ac21d874345f943530c122"}, - {file = "coverage-6.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a82d79586a0a4f5fd1cf153e647464ced402938fbccb3ffc358c7babd4da1dd9"}, - {file = "coverage-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3be1206dc09fb6298de3fce70593e27436862331a85daee36270b6d0e1c251c4"}, - {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9cd3828bbe1a40070c11fe16a51df733fd2f0cb0d745fb83b7b5c1f05967df7"}, - {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d036dc1ed8e1388e995833c62325df3f996675779541f682677efc6af71e96cc"}, - {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:04560539c19ec26995ecfb3d9307ff154fbb9a172cb57e3b3cfc4ced673103d1"}, - {file = "coverage-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:e4fb7ced4d9dec77d6cf533acfbf8e1415fe799430366affb18d69ee8a3c6330"}, - {file = "coverage-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:77b1da5767ed2f44611bc9bc019bc93c03fa495728ec389759b6e9e5039ac6b1"}, - {file = "coverage-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:61b598cbdbaae22d9e34e3f675997194342f866bb1d781da5d0be54783dce1ff"}, - {file = "coverage-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36e9040a43d2017f2787b28d365a4bb33fcd792c7ff46a047a04094dc0e2a30d"}, - {file = "coverage-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9f1627e162e3864a596486774876415a7410021f4b67fd2d9efdf93ade681afc"}, - {file = "coverage-6.0.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7a0b42db2a47ecb488cde14e0f6c7679a2c5a9f44814393b162ff6397fcdfbb"}, - {file = "coverage-6.0.2-cp38-cp38-win32.whl", hash = "sha256:a1b73c7c4d2a42b9d37dd43199c5711d91424ff3c6c22681bc132db4a4afec6f"}, - {file = "coverage-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:1db67c497688fd4ba85b373b37cc52c50d437fd7267520ecd77bddbd89ea22c9"}, - {file = "coverage-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f2f184bf38e74f152eed7f87e345b51f3ab0b703842f447c22efe35e59942c24"}, - {file = "coverage-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1cf1deb3d5544bd942356364a2fdc8959bad2b6cf6eb17f47d301ea34ae822"}, - {file = "coverage-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ad9b8c1206ae41d46ec7380b78ba735ebb77758a650643e841dd3894966c31d0"}, - {file = "coverage-6.0.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:381d773d896cc7f8ba4ff3b92dee4ed740fb88dfe33b6e42efc5e8ab6dfa1cfe"}, - {file = "coverage-6.0.2-cp39-cp39-win32.whl", hash = "sha256:424c44f65e8be58b54e2b0bd1515e434b940679624b1b72726147cfc6a9fc7ce"}, - {file = "coverage-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:abbff240f77347d17306d3201e14431519bf64495648ca5a49571f988f88dee9"}, - {file = "coverage-6.0.2-pp36-none-any.whl", hash = "sha256:7092eab374346121805fb637572483270324407bf150c30a3b161fc0c4ca5164"}, - {file = "coverage-6.0.2-pp37-none-any.whl", hash = "sha256:30922626ce6f7a5a30bdba984ad21021529d3d05a68b4f71ea3b16bda35b8895"}, - {file = "coverage-6.0.2.tar.gz", hash = "sha256:6807947a09510dc31fa86f43595bf3a14017cd60bf633cc746d52141bfa6b149"}, + {file = "coverage-6.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:675adb3b3380967806b3cbb9c5b00ceb29b1c472692100a338730c1d3e59c8b9"}, + {file = "coverage-6.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95a58336aa111af54baa451c33266a8774780242cab3704b7698d5e514840758"}, + {file = "coverage-6.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d0a595a781f8e186580ff8e3352dd4953b1944289bec7705377c80c7e36c4d6c"}, + {file = "coverage-6.1.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d3c5f49ce6af61154060640ad3b3281dbc46e2e0ef2fe78414d7f8a324f0b649"}, + {file = "coverage-6.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:310c40bed6b626fd1f463e5a83dba19a61c4eb74e1ac0d07d454ebbdf9047e9d"}, + {file = "coverage-6.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a4d48e42e17d3de212f9af44f81ab73b9378a4b2b8413fd708d0d9023f2bbde4"}, + {file = "coverage-6.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ffa545230ca2ad921ad066bf8fd627e7be43716b6e0fcf8e32af1b8188ccb0ab"}, + {file = "coverage-6.1.2-cp310-cp310-win32.whl", hash = "sha256:cd2d11a59afa5001ff28073ceca24ae4c506da4355aba30d1e7dd2bd0d2206dc"}, + {file = "coverage-6.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:96129e41405887a53a9cc564f960d7f853cc63d178f3a182fdd302e4cab2745b"}, + {file = "coverage-6.1.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1de9c6f5039ee2b1860b7bad2c7bc3651fbeb9368e4c4d93e98a76358cdcb052"}, + {file = "coverage-6.1.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:80cb70264e9a1d04b519cdba3cd0dc42847bf8e982a4d55c769b9b0ee7cdce1e"}, + {file = "coverage-6.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:ba6125d4e55c0b8e913dad27b22722eac7abdcb1f3eab1bd090eee9105660266"}, + {file = "coverage-6.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8492d37acdc07a6eac6489f6c1954026f2260a85a4c2bb1e343fe3d35f5ee21a"}, + {file = "coverage-6.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:66af99c7f7b64d050d37e795baadf515b4561124f25aae6e1baa482438ecc388"}, + {file = "coverage-6.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ebcc03e1acef4ff44f37f3c61df478d6e469a573aa688e5a162f85d7e4c3860d"}, + {file = "coverage-6.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98d44a8136eebbf544ad91fef5bd2b20ef0c9b459c65a833c923d9aa4546b204"}, + {file = "coverage-6.1.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c18725f3cffe96732ef96f3de1939d81215fd6d7d64900dcc4acfe514ea4fcbf"}, + {file = "coverage-6.1.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c8e9c4bcaaaa932be581b3d8b88b677489975f845f7714efc8cce77568b6711c"}, + {file = "coverage-6.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:06d009e8a29483cbc0520665bc46035ffe9ae0e7484a49f9782c2a716e37d0a0"}, + {file = "coverage-6.1.2-cp36-cp36m-win32.whl", hash = "sha256:e5432d9c329b11c27be45ee5f62cf20a33065d482c8dec1941d6670622a6fb8f"}, + {file = "coverage-6.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:82fdcb64bf08aa5db881db061d96db102c77397a570fbc112e21c48a4d9cb31b"}, + {file = "coverage-6.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:94f558f8555e79c48c422045f252ef41eb43becdd945e9c775b45ebfc0cbd78f"}, + {file = "coverage-6.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:046647b96969fda1ae0605f61288635209dd69dcd27ba3ec0bf5148bc157f954"}, + {file = "coverage-6.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cc799916b618ec9fd00135e576424165691fec4f70d7dc12cfaef09268a2478c"}, + {file = "coverage-6.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:62646d98cf0381ffda301a816d6ac6c35fc97aa81b09c4c52d66a15c4bef9d7c"}, + {file = "coverage-6.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:27a3df08a855522dfef8b8635f58bab81341b2fb5f447819bc252da3aa4cf44c"}, + {file = "coverage-6.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:610c0ba11da8de3a753dc4b1f71894f9f9debfdde6559599f303286e70aeb0c2"}, + {file = "coverage-6.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:35b246ae3a2c042dc8f410c94bcb9754b18179cdb81ff9477a9089dbc9ecc186"}, + {file = "coverage-6.1.2-cp37-cp37m-win32.whl", hash = "sha256:0cde7d9fe2fb55ff68ebe7fb319ef188e9b88e0a3d1c9c5db7dd829cd93d2193"}, + {file = "coverage-6.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:958ac66272ff20e63d818627216e3d7412fdf68a2d25787b89a5c6f1eb7fdd93"}, + {file = "coverage-6.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a300b39c3d5905686c75a369d2a66e68fd01472ea42e16b38c948bd02b29e5bd"}, + {file = "coverage-6.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d3855d5d26292539861f5ced2ed042fc2aa33a12f80e487053aed3bcb6ced13"}, + {file = "coverage-6.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586d38dfc7da4a87f5816b203ff06dd7c1bb5b16211ccaa0e9788a8da2b93696"}, + {file = "coverage-6.1.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a34fccb45f7b2d890183a263578d60a392a1a218fdc12f5bce1477a6a68d4373"}, + {file = "coverage-6.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:bc1ee1318f703bc6c971da700d74466e9b86e0c443eb85983fb2a1bd20447263"}, + {file = "coverage-6.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3f546f48d5d80a90a266769aa613bc0719cb3e9c2ef3529d53f463996dd15a9d"}, + {file = "coverage-6.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd92ece726055e80d4e3f01fff3b91f54b18c9c357c48fcf6119e87e2461a091"}, + {file = "coverage-6.1.2-cp38-cp38-win32.whl", hash = "sha256:24ed38ec86754c4d5a706fbd5b52b057c3df87901a8610d7e5642a08ec07087e"}, + {file = "coverage-6.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:97ef6e9119bd39d60ef7b9cd5deea2b34869c9f0b9777450a7e3759c1ab09b9b"}, + {file = "coverage-6.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e5a8c947a2a89c56655ecbb789458a3a8e3b0cbf4c04250331df8f647b3de59"}, + {file = "coverage-6.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a39590d1e6acf6a3c435c5d233f72f5d43b585f5be834cff1f21fec4afda225"}, + {file = "coverage-6.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9d2c2e3ce7b8cc932a2f918186964bd44de8c84e2f9ef72dc616f5bb8be22e71"}, + {file = "coverage-6.1.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3348865798c077c695cae00da0924136bb5cc501f236cfd6b6d9f7a3c94e0ec4"}, + {file = "coverage-6.1.2-cp39-cp39-win32.whl", hash = "sha256:fae3fe111670e51f1ebbc475823899524e3459ea2db2cb88279bbfb2a0b8a3de"}, + {file = "coverage-6.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:af45eea024c0e3a25462fade161afab4f0d9d9e0d5a5d53e86149f74f0a35ecc"}, + {file = "coverage-6.1.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:eab14fdd410500dae50fd14ccc332e65543e7b39f6fc076fe90603a0e5d2f929"}, + {file = "coverage-6.1.2.tar.gz", hash = "sha256:d9a635114b88c0ab462e0355472d00a180a5fbfd8511e7f18e4ac32652e7d972"}, +] +coveralls = [ + {file = "coveralls-3.3.1-py2.py3-none-any.whl", hash = "sha256:f42015f31d386b351d4226389b387ae173207058832fbf5c8ec4b40e27b16026"}, + {file = "coveralls-3.3.1.tar.gz", hash = "sha256:b32a8bb5d2df585207c119d6c01567b81fba690c9c10a753bfe27a335bfc43ea"}, ] "discord.py" = [] distlib = [ {file = "distlib-0.3.2-py2.py3-none-any.whl", hash = "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c"}, {file = "distlib-0.3.2.zip", hash = "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736"}, ] +docopt = [ + {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, +] execnet = [ {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, diff --git a/pyproject.toml b/pyproject.toml index 245c8fad..7387889b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,8 @@ flake8-todo = "~=0.7" isort = "^5.9.2" pep8-naming = "~=0.11" # testing -codecov = "^2.1.11" coverage = { extras = ["toml"], version = "^6.0.2" } +coveralls = "^3.3.1" pytest = "^6.2.4" pytest-asyncio = "^0.15.1" pytest-cov = "^3.0.0" @@ -81,7 +81,6 @@ include = '\.pyi?$' [tool.taskipy.tasks] start = { cmd = "python -m modmail", help = "Run bot" } black = { cmd = "black --check .", help = "dry run of black" } -codecov-validate = { cmd = "curl --data-binary @.codecov.yml https://codecov.io/validate", help = "Validate `.codecov.yml` with their api." } cov-server = { cmd = "coverage html", help = "Start an http.server for viewing coverage data." } post_cov-server = "python -m http.server 8012 --bind 127.0.0.1 --directory htmlcov" docs = { cmd = "mkdocs serve", help = "Run the docs on a local automatically reloading server" } From 8c6c9420114a2da123c40a86f076a03cdde8704b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 24 Nov 2021 15:44:23 -0500 Subject: [PATCH 72/99] chore: collect coverage for all test files --- .github/workflows/lint_test.yml | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index 7b520cc5..5b02bfb0 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -162,7 +162,8 @@ jobs: # This is saved to ./.coverage to be used by codecov to link a # coverage report to github. - name: Run tests and generate coverage report - run: python -m pytest -n auto --dist loadfile --cov --disable-warnings -q + id: run_tests + run: python -m pytest tests -n auto --dist loadfile --cov --disable-warnings -q # This step will publish the coverage reports to coveralls.io and # print a "job" link in the output of the GitHub Action diff --git a/pyproject.toml b/pyproject.toml index 7387889b..7ed75e6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ build-backend = "poetry.core.masonry.api" [tool.coverage.run] branch = true -source_pkgs = ['modmail', 'tests.modmail'] +source_pkgs = ['modmail', 'tests'] omit = ["modmail/plugins/**.*"] [tool.pytest.ini_options] From 672fcd029caf2a66f314e2cc6c6c1ef146efa0cf Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 24 Nov 2021 15:51:10 -0500 Subject: [PATCH 73/99] chore: remove coverage html artifacts --- .github/workflows/lint_test.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index 5b02bfb0..7308b96e 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -175,16 +175,6 @@ jobs: COVERALLS_SERVICE_NAME: github run: python -m coveralls - - name: Coverage HTML - run: python -m coverage html - - - name: Upload coverage as artifact - uses: actions/upload-artifact@v2 - with: - name: coverage-${{ runner.os }}-python-${{ env.PYTHON_VERSION }} - path: htmlcov - - coveralls: name: Indicate completion to coveralls.io needs: test From 36f5ce19c692d5f089842b3f20d9a1983a1a6329 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 24 Nov 2021 15:59:51 -0500 Subject: [PATCH 74/99] fix: publish coverage on a failed test run and make coveralls optional --- .github/workflows/lint_test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index 7308b96e..706079ca 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -168,6 +168,8 @@ jobs: # This step will publish the coverage reports to coveralls.io and # print a "job" link in the output of the GitHub Action - name: Publish coverage report to coveralls.io + if: always() && (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure') + continue-on-error: true env: COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: coverage-${{ runner.os }}-python-${{ env.PYTHON_VERSION }} @@ -177,16 +179,20 @@ jobs: coveralls: name: Indicate completion to coveralls.io - needs: test runs-on: ubuntu-latest + needs: test + continue-on-error: true + # we always want to ensure we attempt to send a finish to coveralls + if: always() container: python:3-slim steps: - name: Coveralls Finished + continue-on-error: true env: COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_SERVICE_NAME: github run: | - python3 -m pip install --upgrade coveralls + python3 -m pip install --upgrade coveralls -qqq python3 -m coveralls --finish artifact: From f0ec0674f44886612609812e949ab6b9ebd76401 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 25 Nov 2021 10:56:05 -0500 Subject: [PATCH 75/99] fix: use type() instead of __class__ --- modmail/extensions/configuration_manager.py | 4 ++-- scripts/export_new_config_to_default_config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index eee1b932..b68d152b 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -146,7 +146,7 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: # depending on common problems, it is *possible* to add `_` but would require fuzzy matching over # all of the keys since that can also be a valid character name. - fields = get_all_conf_options(config.default().__class__) + fields = get_all_conf_options(type(config.default())) new_arg = "" for c in arg.lower(): @@ -172,7 +172,7 @@ class ConfigurationManager(ModmailCog, name="Configuration Manager"): def __init__(self, bot: ModmailBot): self.bot = bot - self.config_fields = get_all_conf_options(config.default().__class__) + self.config_fields = get_all_conf_options(type(config.default())) @commands.group(name="config", aliases=("cfg", "conf"), invoke_without_command=True) async def config_group(self, ctx: Context) -> None: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 854e1f00..203d8cf6 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -210,7 +210,7 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada ENV_EXPORT_FILE.unlink(missing_ok=True) ENV_EXPORT_FILE.touch() - exported = get_env_vars(default.__class__) + exported = get_env_vars(type(default)) app_json_env = dict() From a0fb30a74fe8d1424147f03986a75269bad7de16 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 26 Nov 2021 21:38:44 -0500 Subject: [PATCH 76/99] chore: comment coveralls actions --- .github/workflows/lint_test.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index 706079ca..ff6f8d51 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -168,7 +168,10 @@ jobs: # This step will publish the coverage reports to coveralls.io and # print a "job" link in the output of the GitHub Action - name: Publish coverage report to coveralls.io + # upload coverage even if a test run failed + # this is a test, and may be removed in the future if: always() && (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure') + # important that we don't fail the workflow when coveralls is down continue-on-error: true env: COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -177,22 +180,31 @@ jobs: COVERALLS_SERVICE_NAME: github run: python -m coveralls - coveralls: + coveralls-finish: name: Indicate completion to coveralls.io runs-on: ubuntu-latest needs: test + # we don't want to fail the workflow when coveralls is down continue-on-error: true # we always want to ensure we attempt to send a finish to coveralls if: always() - container: python:3-slim steps: + # Set up a consistent version of Python + - name: Set up Python 3.9 + id: python + uses: actions/setup-python@v2 + with: + python-version: '3.9' - name: Coveralls Finished continue-on-error: true env: COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_SERVICE_NAME: github + # NOTE: this has a small thing where this will not always be the same with the poetry.lock file + # given how this is installed for one api request, its not worth pinning to me. + # any bugs caused by this can be solved when they occur run: | - python3 -m pip install --upgrade coveralls -qqq + python3 -m pip install --upgrade coveralls python3 -m coveralls --finish artifact: From 763a2438a3a94d86506414309438d456f22cf96f Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 25 Nov 2021 13:14:39 -0500 Subject: [PATCH 77/99] fix: run black and flake8 even on pre-commit failure --- .github/workflows/lint_test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index ff6f8d51..4a7bceed 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -92,11 +92,14 @@ jobs: # black and flake8 action. As pre-commit does not support user installs, # we set PIP_USER=0 to not do a user install. - name: Run pre-commit hooks + id: pre-commit run: export PIP_USER=0; SKIP="no-commit-to-branch,black,flake8" pre-commit run --all-files # Run black seperately as we don't want to reformat the files # just error if something isn't formatted correctly. - name: Check files with black + id: black + if: always() && (steps.pre-commit.outcome == 'success' || steps.pre-commit.outcome == 'failure') run: black . --check --diff --color # Run flake8 and have it format the linting errors in the format of @@ -108,6 +111,8 @@ jobs: # Format used: # ::error file={filename},line={line},col={col}::{message} - name: Run flake8 + id: flake8 + if: always() && (steps.pre-commit.outcome == 'success' || steps.pre-commit.outcome == 'failure') run: "flake8 \ --format='::error file=%(path)s,line=%(row)d,col=%(col)d::\ [flake8] %(code)s: %(text)s'" From 29617aa7a372ecfc4d184e98342eb3c1ba7708eb Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 18 Nov 2021 04:11:25 -0500 Subject: [PATCH 78/99] chore: fix windows dns lookup fix: use WindowsSelectorEventLoopPolicy on windows --- modmail/__init__.py | 6 ++++++ modmail/bot.py | 41 ++++++++++++++++++++++++++++++++++------- tests/conftest.py | 16 ++++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/modmail/__init__.py b/modmail/__init__.py index 9377ffd2..58403bd6 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -1,5 +1,7 @@ +import asyncio import logging import logging.handlers +import os from pathlib import Path import coloredlogs @@ -7,6 +9,10 @@ from modmail.log import ModmailLogger +# On Windows, the selector event loop is required for aiodns. +if os.name == "nt": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + logging.TRACE = 5 logging.NOTICE = 25 logging.addLevelName(logging.TRACE, "TRACE") diff --git a/modmail/bot.py b/modmail/bot.py index de27ea08..06de9ea1 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -1,12 +1,12 @@ import asyncio import logging import signal +import socket import typing as t -from typing import Any +import aiohttp import arrow import discord -from aiohttp import ClientSession from discord import Activity, AllowedMentions, Intents from discord.client import _cleanup_loop from discord.ext import commands @@ -41,9 +41,12 @@ class ModmailBot(commands.Bot): def __init__(self, **kwargs): self.config = CONFIG self.start_time: t.Optional[arrow.Arrow] = None # arrow.utcnow() - self.http_session: t.Optional[ClientSession] = None + self.http_session: t.Optional[aiohttp.ClientSession] = None self.dispatcher = Dispatcher() + self._connector = None + self._resolver = None + status = discord.Status.online activity = Activity(type=discord.ActivityType.listening, name="users dming me!") # listen to messages mentioning the bot or matching the prefix @@ -65,6 +68,24 @@ def __init__(self, **kwargs): **kwargs, ) + async def create_connectors(self, *args, **kwargs) -> None: + """Re-create the connector and set up sessions before logging into Discord.""" + # Use asyncio for DNS resolution instead of threads so threads aren't spammed. + self._resolver = aiohttp.AsyncResolver() + + # Use AF_INET as its socket family to prevent HTTPS related problems both locally + # and in production. + self._connector = aiohttp.TCPConnector( + resolver=self._resolver, + family=socket.AF_INET, + ) + + # Client.login() will call HTTPClient.static_login() which will create a session using + # this connector attribute. + self.http.connector = self._connector + + self.http_session = aiohttp.ClientSession(connector=self._connector) + async def start(self, token: str, reconnect: bool = True) -> None: """ Start the bot. @@ -74,8 +95,8 @@ async def start(self, token: str, reconnect: bool = True) -> None: """ try: # create the aiohttp session - self.http_session = ClientSession(loop=self.loop) - self.logger.trace("Created ClientSession.") + await self.create_connectors() + self.logger.trace("Created aiohttp.ClientSession.") # set start time to when we started the bot. # This is now, since we're about to connect to the gateway. # This should also be before we load any extensions, since if they have a load time, it should @@ -122,7 +143,7 @@ def run(self, *args, **kwargs) -> None: except NotImplementedError: pass - def stop_loop_on_completion(f: Any) -> None: + def stop_loop_on_completion(f: t.Any) -> None: loop.stop() future = asyncio.ensure_future(self.start(*args, **kwargs), loop=loop) @@ -164,10 +185,16 @@ async def close(self) -> None: except Exception: self.logger.error(f"Exception occured while removing cog {cog.name}", exc_info=True) + await super().close() + if self.http_session: await self.http_session.close() - await super().close() + if self._connector: + await self._connector.close() + + if self._resolver: + await self._resolver.close() def load_extensions(self) -> None: """Load all enabled extensions.""" diff --git a/tests/conftest.py b/tests/conftest.py index 5871ed8e..9ac88d6c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,17 @@ +import aiohttp import pytest + + +@pytest.fixture +@pytest.mark.asyncio +async def http_session() -> aiohttp.ClientSession: + """Fixture function for a aiohttp.ClientSession.""" + resolver = aiohttp.AsyncResolver() + connector = aiohttp.TCPConnector(resolver=resolver) + client_session = aiohttp.ClientSession(connector=connector) + + yield client_session + + await client_session.close() + await connector.close() + await resolver.close() From 5cfaf7b334b5fde77b9a730cd4cd2caaed74ed20 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 19 Nov 2021 01:22:35 -0500 Subject: [PATCH 79/99] tests: add aioresponses for aiohttp response mocking --- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + tests/conftest.py | 16 ++++++++++++++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8f4c1718..c1ca0ddd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,6 +31,17 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["aiodns", "brotlipy", "cchardet"] +[[package]] +name = "aioresponses" +version = "0.7.2" +description = "Mock out requests made by ClientSession from aiohttp package" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +aiohttp = ">=2.0.0,<4.0.0" + [[package]] name = "arrow" version = "1.1.1" @@ -1218,7 +1229,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "1086984557d7a4f7503e1f68d7a3a5662d2960998fec490144a3debdcc259256" +content-hash = "0d2af2b4ba03d4d2292cd59fdb9aaa33b32b49d79f130a2f6d4419d7903c6fbc" [metadata.files] aiodns = [ @@ -1264,6 +1275,10 @@ aiohttp = [ {file = "aiohttp-3.7.4.post0-cp39-cp39-win_amd64.whl", hash = "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe"}, {file = "aiohttp-3.7.4.post0.tar.gz", hash = "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf"}, ] +aioresponses = [ + {file = "aioresponses-0.7.2-py2.py3-none-any.whl", hash = "sha256:2f8ff624543066eb465b0238de68d29231e8488f41dc4b5a9dae190982cdae50"}, + {file = "aioresponses-0.7.2.tar.gz", hash = "sha256:82e495d118b74896aa5b4d47e17effb5e2cc783e510ae395ceade5e87cabe89a"}, +] arrow = [ {file = "arrow-1.1.1-py3-none-any.whl", hash = "sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510"}, {file = "arrow-1.1.1.tar.gz", hash = "sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a"}, diff --git a/pyproject.toml b/pyproject.toml index 7ed75e6e..e8f17f06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ flake8-todo = "~=0.7" isort = "^5.9.2" pep8-naming = "~=0.11" # testing +aioresponses = "^0.7.2" coverage = { extras = ["toml"], version = "^6.0.2" } coveralls = "^3.3.1" pytest = "^6.2.4" diff --git a/tests/conftest.py b/tests/conftest.py index 9ac88d6c..40c3ae9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,23 @@ import aiohttp +import aioresponses import pytest +@pytest.fixture +def aioresponse(): + """Fixture to mock aiohttp responses.""" + with aioresponses.aioresponses() as aioresponse: + yield aioresponse + + @pytest.fixture @pytest.mark.asyncio -async def http_session() -> aiohttp.ClientSession: - """Fixture function for a aiohttp.ClientSession.""" +async def http_session(aioresponse) -> aiohttp.ClientSession: + """ + Fixture function for a aiohttp.ClientSession. + + Requests fixture aioresponse to ensure that all client sessions do not make actual requests. + """ resolver = aiohttp.AsyncResolver() connector = aiohttp.TCPConnector(resolver=resolver) client_session = aiohttp.ClientSession(connector=connector) From 7ba1c008d1c3ad8381922e145d0da6cf84c67cbe Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 09:21:51 -0500 Subject: [PATCH 80/99] tests: add tests for aiohttp fixture --- tests/test_fixtures.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_fixtures.py diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py new file mode 100644 index 00000000..906c3301 --- /dev/null +++ b/tests/test_fixtures.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import aiohttp +import pytest + + +if TYPE_CHECKING: + import aioresponses + + +class TestSessionFixture: + """Grouping for aiohttp.ClientSession fixture tests.""" + + @pytest.mark.asyncio + async def test_session_fixture_no_requests(self, http_session: aiohttp.ClientSession): + """ + Test all requests fail. + + This means that aioresponses is being requested by the http_session fixture. + """ + url = "https://github.com/" + + with pytest.raises(aiohttp.ClientConnectionError): + await http_session.get(url) + + @pytest.mark.asyncio + async def test_session_fixture_mock_requests( + self, aioresponse: aioresponses.aioresponses, http_session: aiohttp.ClientSession + ): + """ + Test all requests fail. + + This means that aioresponses is being requested by the http_session fixture. + """ + url = "https://github.com/" + status = 200 + aioresponse.get(url, status=status) + + async with http_session.get(url) as resp: + assert status == resp.status From ebe36f7756191f2b601e5fae4ca0292adecfa92c Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 12:20:54 -0500 Subject: [PATCH 81/99] chore: better document why the loop policy is required --- modmail/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modmail/__init__.py b/modmail/__init__.py index 58403bd6..c99c962e 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -9,7 +9,9 @@ from modmail.log import ModmailLogger -# On Windows, the selector event loop is required for aiodns. +# on windows aiodns's asyncio support relies on APIs like add_reader (which aiodns uses) +# are not guaranteed to be available, and in particular are not available when using the +# ProactorEventLoop on Windows, this method is only supported with Windows SelectorEventLoop if os.name == "nt": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) From 25192c52a69794c5e0d8344dff912e41acd2b31e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 26 Nov 2021 22:07:02 -0500 Subject: [PATCH 82/99] nit: uppercase On --- modmail/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/__init__.py b/modmail/__init__.py index c99c962e..1c3966eb 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -9,7 +9,7 @@ from modmail.log import ModmailLogger -# on windows aiodns's asyncio support relies on APIs like add_reader (which aiodns uses) +# On windows aiodns's asyncio support relies on APIs like add_reader (which aiodns uses) # are not guaranteed to be available, and in particular are not available when using the # ProactorEventLoop on Windows, this method is only supported with Windows SelectorEventLoop if os.name == "nt": From 26db296a8c4d43dcce9739a04c1ea83428404f57 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 26 Nov 2021 23:19:50 -0500 Subject: [PATCH 83/99] tools: remove flake8-bandit this hasn't even been configured, so its not doing anything whatsoever as far as I can tell --- poetry.lock | 108 +------------------------------------------------ pyproject.toml | 1 - 2 files changed, 1 insertion(+), 108 deletions(-) diff --git a/poetry.lock b/poetry.lock index c1ca0ddd..936ceab3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -95,21 +95,6 @@ python-versions = ">=2.7" docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3.7)", "pytest-mypy", "pytest-checkdocs (>=2.4)", "pytest-enabler (>=1.0.1)"] -[[package]] -name = "bandit" -version = "1.7.0" -description = "Security oriented static analyser for python code." -category = "dev" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -colorama = {version = ">=0.3.9", markers = "platform_system == \"Windows\""} -GitPython = ">=1.0.1" -PyYAML = ">=5.3.1" -six = ">=1.10.0" -stevedore = ">=1.20.0" - [[package]] name = "black" version = "21.8b0" @@ -344,20 +329,6 @@ python-versions = ">=3.6.1,<4.0.0" [package.dependencies] flake8 = ">=3.7,<4.0" -[[package]] -name = "flake8-bandit" -version = "2.1.2" -description = "Automated security testing with bandit and flake8." -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -bandit = "*" -flake8 = "*" -flake8-polyfill = "*" -pycodestyle = "*" - [[package]] name = "flake8-bugbear" version = "20.11.1" @@ -459,29 +430,6 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["twine", "markdown", "flake8"] -[[package]] -name = "gitdb" -version = "4.0.7" -description = "Git Object Database" -category = "dev" -optional = false -python-versions = ">=3.4" - -[package.dependencies] -smmap = ">=3.0.1,<5" - -[[package]] -name = "gitpython" -version = "3.1.20" -description = "Python Git Library" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -gitdb = ">=4.0.1,<5" -typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\""} - [[package]] name = "humanfriendly" version = "10.0" @@ -711,14 +659,6 @@ category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -[[package]] -name = "pbr" -version = "5.6.0" -description = "Python Build Reasonableness" -category = "dev" -optional = false -python-versions = ">=2.6" - [[package]] name = "pep8-naming" version = "0.12.1" @@ -1073,14 +1013,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -[[package]] -name = "smmap" -version = "4.0.0" -description = "A pure Python implementation of a sliding window memory map manager" -category = "dev" -optional = false -python-versions = ">=3.5" - [[package]] name = "snowballstemmer" version = "2.1.0" @@ -1089,17 +1021,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "stevedore" -version = "3.4.0" -description = "Manage dynamic plugins for Python applications" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pbr = ">=2.0.0,<2.1.0 || >2.1.0" - [[package]] name = "taskipy" version = "1.8.1" @@ -1229,7 +1150,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "0d2af2b4ba03d4d2292cd59fdb9aaa33b32b49d79f130a2f6d4419d7903c6fbc" +content-hash = "aaa5181c8d94a936811266da090c02a531780a7beaf35985198b82a06f5b306b" [metadata.files] aiodns = [ @@ -1299,10 +1220,6 @@ attrs = [ {file = "backports.entry_points_selectable-1.1.0-py2.py3-none-any.whl", hash = "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc"}, {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, ] -bandit = [ - {file = "bandit-1.7.0-py3-none-any.whl", hash = "sha256:216be4d044209fa06cf2a3e51b319769a51be8318140659719aa7a115c35ed07"}, - {file = "bandit-1.7.0.tar.gz", hash = "sha256:8a4c7415254d75df8ff3c3b15cfe9042ecee628a1e40b44c15a98890fbfc2608"}, -] black = [ {file = "black-21.8b0-py3-none-any.whl", hash = "sha256:2a0f9a8c2b2a60dbcf1ccb058842fb22bdbbcb2f32c6cc02d9578f90b92ce8b7"}, {file = "black-21.8b0.tar.gz", hash = "sha256:570608d28aa3af1792b98c4a337dbac6367877b47b12b88ab42095cfc1a627c2"}, @@ -1534,9 +1451,6 @@ flake8-annotations = [ {file = "flake8-annotations-2.6.2.tar.gz", hash = "sha256:0d6cd2e770b5095f09689c9d84cc054c51b929c41a68969ea1beb4b825cac515"}, {file = "flake8_annotations-2.6.2-py3-none-any.whl", hash = "sha256:d10c4638231f8a50c0a597c4efce42bd7b7d85df4f620a0ddaca526138936a4f"}, ] -flake8-bandit = [ - {file = "flake8_bandit-2.1.2.tar.gz", hash = "sha256:687fc8da2e4a239b206af2e54a90093572a60d0954f3054e23690739b0b0de3b"}, -] flake8-bugbear = [ {file = "flake8-bugbear-20.11.1.tar.gz", hash = "sha256:528020129fea2dea33a466b9d64ab650aa3e5f9ffc788b70ea4bc6cf18283538"}, {file = "flake8_bugbear-20.11.1-py36.py37.py38-none-any.whl", hash = "sha256:f35b8135ece7a014bc0aee5b5d485334ac30a6da48494998cc1fabf7ec70d703"}, @@ -1567,14 +1481,6 @@ flake8-todo = [ ghp-import = [ {file = "ghp-import-2.0.1.tar.gz", hash = "sha256:753de2eace6e0f7d4edfb3cce5e3c3b98cd52aadb80163303d1d036bda7b4483"}, ] -gitdb = [ - {file = "gitdb-4.0.7-py3-none-any.whl", hash = "sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0"}, - {file = "gitdb-4.0.7.tar.gz", hash = "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"}, -] -gitpython = [ - {file = "GitPython-3.1.20-py3-none-any.whl", hash = "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a"}, - {file = "GitPython-3.1.20.tar.gz", hash = "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519"}, -] humanfriendly = [ {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, @@ -1781,10 +1687,6 @@ pathspec = [ {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] -pbr = [ - {file = "pbr-5.6.0-py2.py3-none-any.whl", hash = "sha256:c68c661ac5cc81058ac94247278eeda6d2e6aecb3e227b0387c30d277e7ef8d4"}, - {file = "pbr-5.6.0.tar.gz", hash = "sha256:42df03e7797b796625b1029c0400279c7c34fd7df24a7d7818a1abb5b38710dd"}, -] pep8-naming = [ {file = "pep8-naming-0.12.1.tar.gz", hash = "sha256:bb2455947757d162aa4cad55dba4ce029005cd1692f2899a21d51d8630ca7841"}, {file = "pep8_naming-0.12.1-py2.py3-none-any.whl", hash = "sha256:4a8daeaeb33cfcde779309fc0c9c0a68a3bbe2ad8a8308b763c5068f86eb9f37"}, @@ -2044,18 +1946,10 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -smmap = [ - {file = "smmap-4.0.0-py2.py3-none-any.whl", hash = "sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2"}, - {file = "smmap-4.0.0.tar.gz", hash = "sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182"}, -] snowballstemmer = [ {file = "snowballstemmer-2.1.0-py2.py3-none-any.whl", hash = "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2"}, {file = "snowballstemmer-2.1.0.tar.gz", hash = "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914"}, ] -stevedore = [ - {file = "stevedore-3.4.0-py3-none-any.whl", hash = "sha256:920ce6259f0b2498aaa4545989536a27e4e4607b8318802d7ddc3a533d3d069e"}, - {file = "stevedore-3.4.0.tar.gz", hash = "sha256:59b58edb7f57b11897f150475e7bc0c39c5381f0b8e3fa9f5c20ce6c89ec4aa1"}, -] taskipy = [ {file = "taskipy-1.8.1-py3-none-any.whl", hash = "sha256:2b98f499966e40175d1f1306a64587f49dfa41b90d0d86c8f28b067cc58d0a56"}, {file = "taskipy-1.8.1.tar.gz", hash = "sha256:7a2404125817e45d80e13fa663cae35da6e8ba590230094e815633653e25f98f"}, diff --git a/pyproject.toml b/pyproject.toml index e8f17f06..57d3a124 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ taskipy = "^1.6.0" black = "^21.7b0" flake8 = "~=3.8" flake8-annotations = "~=2.3" -flake8-bandit = "^2.1.2" flake8-bugbear = "~=20.1" flake8-docstrings = "~=1.5" flake8-isort = "^4.0.0" From 7347f592323f32672dc5e794d442ff643788fbef Mon Sep 17 00:00:00 2001 From: Shivansh-007 Date: Sat, 27 Nov 2021 12:30:20 +0530 Subject: [PATCH 84/99] Add lock no update task (#133) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 57d3a124..c2d4b071 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ docs = { cmd = "mkdocs serve", help = "Run the docs on a local automatically rel export = { cmd = "python -m scripts.export_requirements", help = "Export the requirements from poetry.lock to requirements.txt" } flake8 = { cmd = "python -m flake8", help = "Lints code with flake8" } lint = { cmd = "pre-commit run --all-files", help = "Checks all files for CI errors" } +lock = { cmd = 'poetry lock --no-update && task export --docs', help = 'Relock the dependencies without updating them. Also runs the export scripts' } precommit = { cmd = "pre-commit install --install-hooks", help = "Installs the precommit hook" } report = { cmd = "coverage report", help = "Show coverage report from previously run tests." } scripts = { cmd = 'python -m scripts', help = 'Run the scripts wrapper cli.' } From 9ff011ff2ba551fe4d1748024969c986681fb775 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 24 Sep 2021 10:29:08 -0400 Subject: [PATCH 85/99] fix: paginator can now not use an embed --- modmail/utils/pagination.py | 48 ++++++++++++++++++++------ tests/modmail/utils/test_pagination.py | 3 +- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index e0f633a4..0c381ea6 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -33,6 +33,8 @@ logger: ModmailLogger = logging.getLogger(__name__) +_AUTOGENERATE = object() + class ButtonPaginator(ui.View, DpyPaginator): """ @@ -57,7 +59,7 @@ def __init__( contents: Union[List[str], str], /, source_message: Optional[discord.Message] = None, - embed: Embed = None, + embed: Optional[Embed] = _AUTOGENERATE, timeout: float = 180, *, footer_text: str = None, @@ -82,7 +84,13 @@ def __init__( self.suffix = suffix self.max_size = max_size self.linesep = linesep - self.embed = embed or Embed() + if embed is _AUTOGENERATE: + self.embed = Embed() + else: + self.embed = embed + + # used if embed is None + self.content = "" # temporary to support strings as contents. This will be changed when we added wrapping. if isinstance(contents, str): @@ -116,8 +124,8 @@ def __init__( # set footer to embed.footer if embed is set # this is because we will be modifying the footer of this embed - if embed is not None: - if not isinstance(embed.footer, EmbedProxy) and footer_text is None: + if self.embed is not None: + if not isinstance(self.embed.footer, EmbedProxy) and footer_text is None: footer_text = embed.footer self.footer_text = footer_text self.clear() @@ -140,7 +148,7 @@ async def paginate( source_message: discord.Message = None, /, timeout: float = 180, - embed: Embed = None, + embed: Embed = _AUTOGENERATE, *, footer_text: str = None, only: Optional[discord.abc.User] = None, @@ -178,10 +186,17 @@ async def paginate( channel = source_message.channel paginator.update_states() - paginator.embed.description = paginator.pages[paginator.index] + if paginator.embed: + paginator.embed.description = paginator.pages[paginator.index] + else: + paginator.content = paginator.pages[paginator.index] # if there's only one page, don't send the view if len(paginator.pages) < 2: - await channel.send(embeds=[paginator.embed]) + if paginator.embed: + await channel.send(embeds=[paginator.embed]) + else: + await channel.send(content=paginator.content) + return if len(paginator.pages) < (show_jump_buttons_min_pages or 3): @@ -189,7 +204,10 @@ async def paginate( if getattr(item, "custom_id", None) in ["pag_jump_first", "pag_jump_last"]: paginator.remove_item(item) - msg: discord.Message = await channel.send(embeds=[paginator.embed], view=paginator) + if paginator.embed is None: + msg: discord.Message = await channel.send(content=paginator.content, view=paginator) + else: + msg: discord.Message = await channel.send(embeds=[paginator.embed], view=paginator) await paginator.wait() await msg.edit(view=None) @@ -212,8 +230,11 @@ async def interaction_check(self, interaction: Interaction) -> bool: ) return False - def get_footer(self) -> str: + def get_footer(self) -> Optional[str]: """Returns the footer text.""" + if self.embed is None: + self.content = self._pages[self.index] + return None self.embed.description = self._pages[self.index] page_indicator = f"Page {self.index+1}/{len(self._pages)}" footer_txt = ( @@ -230,7 +251,9 @@ def update_states(self) -> None: if the paginator is on the last page, the jump last/move forward buttons will be disabled. """ # update the footer - self.embed.set_footer(text=self.get_footer()) + text = self.get_footer() + if self.embed: + self.embed.set_footer(text=text) # determine if the jump buttons should be enabled more_than_two_pages = len(self._pages) > 2 @@ -264,7 +287,10 @@ async def send_page(self, interaction: Interaction) -> None: """Send new page to discord, after updating the view to have properly disabled buttons.""" self.update_states() - await interaction.message.edit(embed=self.embed, view=self) + if self.embed: + await interaction.message.edit(embed=self.embed, view=self) + else: + await interaction.message.edit(content=self.content, view=self) @ui.button(label=JUMP_FIRST_LABEL, custom_id="pag_jump_first", style=ButtonStyle.primary) async def go_first(self, _: Button, interaction: Interaction) -> None: diff --git a/tests/modmail/utils/test_pagination.py b/tests/modmail/utils/test_pagination.py index 1993c6ee..562af501 100644 --- a/tests/modmail/utils/test_pagination.py +++ b/tests/modmail/utils/test_pagination.py @@ -27,11 +27,10 @@ async def test_paginator_footer(content: Union[str, List[str]], footer_text: str pag = ButtonPaginator(content, footer_text=footer_text) print("index:", pag.index) print("page len: ", len(pag.pages)) - assert pag.footer_text == footer_text + assert footer_text == pag.footer_text if isinstance(content, str): content = [content] - print(pag.get_footer()) if footer_text is not None: assert pag.get_footer().endswith(f"{len(content)})") assert pag.get_footer().startswith(footer_text) From 5a264439d57f51571e2b9846b681190607503cd0 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 19 Nov 2021 15:29:20 -0500 Subject: [PATCH 86/99] chore: fold get_footer functionality into update_states --- modmail/utils/pagination.py | 27 +++++++++++-------------- tests/modmail/utils/test_pagination.py | 28 -------------------------- 2 files changed, 12 insertions(+), 43 deletions(-) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index 0c381ea6..3ee71ea7 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -230,18 +230,6 @@ async def interaction_check(self, interaction: Interaction) -> bool: ) return False - def get_footer(self) -> Optional[str]: - """Returns the footer text.""" - if self.embed is None: - self.content = self._pages[self.index] - return None - self.embed.description = self._pages[self.index] - page_indicator = f"Page {self.index+1}/{len(self._pages)}" - footer_txt = ( - f"{self.footer_text} ({page_indicator})" if self.footer_text is not None else page_indicator - ) - return footer_txt - def update_states(self) -> None: """ Disable specific components depending on paginator page and length. @@ -251,9 +239,18 @@ def update_states(self) -> None: if the paginator is on the last page, the jump last/move forward buttons will be disabled. """ # update the footer - text = self.get_footer() - if self.embed: - self.embed.set_footer(text=text) + if self.embed is None: + self.content = self._pages[self.index] + else: + self.embed.description = self._pages[self.index] + page_indicator = f"Page {self.index+1}/{len(self._pages)}" + self.embed.set_footer( + text=( + f"{self.footer_text} ({page_indicator})" + if self.footer_text is not None + else page_indicator + ) + ) # determine if the jump buttons should be enabled more_than_two_pages = len(self._pages) > 2 diff --git a/tests/modmail/utils/test_pagination.py b/tests/modmail/utils/test_pagination.py index 562af501..2ada1079 100644 --- a/tests/modmail/utils/test_pagination.py +++ b/tests/modmail/utils/test_pagination.py @@ -1,5 +1,3 @@ -from typing import List, Union - import pytest from modmail.utils.pagination import ButtonPaginator @@ -11,29 +9,3 @@ async def test_paginator_init() -> None: content = ["content"] paginator = ButtonPaginator(content, prefix="", suffix="", linesep="") assert paginator.pages == content - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "content, footer_text", - [ - (["5"], "Snap, crackle, pop"), - (["Earthly"], "world"), - ("There are no plugins installed.", None), - ], -) -async def test_paginator_footer(content: Union[str, List[str]], footer_text: str) -> None: - """Test the paginator footer matches what is passed.""" - pag = ButtonPaginator(content, footer_text=footer_text) - print("index:", pag.index) - print("page len: ", len(pag.pages)) - assert footer_text == pag.footer_text - if isinstance(content, str): - content = [content] - - if footer_text is not None: - assert pag.get_footer().endswith(f"{len(content)})") - assert pag.get_footer().startswith(footer_text) - - else: - assert pag.get_footer().endswith(f"{len(content)}") From c3dacb5ce1dfd1e81fd76142495b5983d5d8b8fe Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 19 Nov 2021 15:51:33 -0500 Subject: [PATCH 87/99] fix: add title and footer field to embedless paginators --- modmail/utils/pagination.py | 39 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index 3ee71ea7..e3de8985 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -66,6 +66,7 @@ def __init__( prefix: str = "```", suffix: str = "```", max_size: int = 2000, + title: str = None, linesep: str = "\n", only_users: Optional[List[Union[discord.Object, discord.abc.User]]] = None, only_roles: Optional[List[Union[discord.Object, discord.Role]]] = None, @@ -91,6 +92,19 @@ def __init__( # used if embed is None self.content = "" + if embed is not None: + if title: + raise TypeError("Cannot set title if embed is None.") + self.title = None + else: + self.title = title + # need to set the max_size down a few to be able to set a "footer" + # page indicator is "page xx of xx" + self.max_size -= 15 + if self.title is not None: + self.max_size -= len(title) + if footer_text is not None: + self.max_size -= len(footer_text) + 1 # temporary to support strings as contents. This will be changed when we added wrapping. if isinstance(contents, str): @@ -157,6 +171,7 @@ async def paginate( prefix: str = "", suffix: str = "", max_size: int = 4000, + title: str = None, linesep: str = "\n", only_users: Optional[List[Union[discord.Object, discord.abc.User]]] = None, only_roles: Optional[List[Union[discord.Object, discord.abc.Role]]] = None, @@ -175,6 +190,7 @@ async def paginate( prefix=prefix, suffix=suffix, max_size=max_size, + title=title, linesep=linesep, only_users=only_users, only_roles=only_roles, @@ -186,10 +202,6 @@ async def paginate( channel = source_message.channel paginator.update_states() - if paginator.embed: - paginator.embed.description = paginator.pages[paginator.index] - else: - paginator.content = paginator.pages[paginator.index] # if there's only one page, don't send the view if len(paginator.pages) < 2: if paginator.embed: @@ -239,18 +251,18 @@ def update_states(self) -> None: if the paginator is on the last page, the jump last/move forward buttons will be disabled. """ # update the footer + page_indicator = f"Page {self.index+1}/{len(self._pages)}" + footer_text = ( + f"{self.footer_text} ({page_indicator})" if self.footer_text is not None else page_indicator + ) if self.embed is None: - self.content = self._pages[self.index] + self.content = (self.title or "") + "\n" + self.content += self._pages[self.index] + self.content += "\n" + footer_text + else: self.embed.description = self._pages[self.index] - page_indicator = f"Page {self.index+1}/{len(self._pages)}" - self.embed.set_footer( - text=( - f"{self.footer_text} ({page_indicator})" - if self.footer_text is not None - else page_indicator - ) - ) + self.embed.set_footer(text=footer_text) # determine if the jump buttons should be enabled more_than_two_pages = len(self._pages) > 2 @@ -287,6 +299,7 @@ async def send_page(self, interaction: Interaction) -> None: if self.embed: await interaction.message.edit(embed=self.embed, view=self) else: + print(len(self.content)) await interaction.message.edit(content=self.content, view=self) @ui.button(label=JUMP_FIRST_LABEL, custom_id="pag_jump_first", style=ButtonStyle.primary) From f00c2e5372819dca200b8cfc4a9997926de3a7a0 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 07:01:54 -0500 Subject: [PATCH 88/99] chore: decompress if statement --- modmail/utils/pagination.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index e3de8985..4c71983a 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -252,9 +252,11 @@ def update_states(self) -> None: """ # update the footer page_indicator = f"Page {self.index+1}/{len(self._pages)}" - footer_text = ( - f"{self.footer_text} ({page_indicator})" if self.footer_text is not None else page_indicator - ) + if self.footer_text: + footer_text = f"{self.footer_text} ({page_indicator})" + else: + footer_text = page_indicator + if self.embed is None: self.content = (self.title or "") + "\n" self.content += self._pages[self.index] From 94da11efbd84277fd98f52ab0457e747c5ec7071 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 2 Dec 2021 00:17:55 -0500 Subject: [PATCH 89/99] fix(paginator): allow titles even if embed is None --- modmail/utils/pagination.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index 4c71983a..877bc2d1 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -92,11 +92,7 @@ def __init__( # used if embed is None self.content = "" - if embed is not None: - if title: - raise TypeError("Cannot set title if embed is None.") - self.title = None - else: + if embed is None: self.title = title # need to set the max_size down a few to be able to set a "footer" # page indicator is "page xx of xx" @@ -301,7 +297,6 @@ async def send_page(self, interaction: Interaction) -> None: if self.embed: await interaction.message.edit(embed=self.embed, view=self) else: - print(len(self.content)) await interaction.message.edit(content=self.content, view=self) @ui.button(label=JUMP_FIRST_LABEL, custom_id="pag_jump_first", style=ButtonStyle.primary) From a20721d0859421beec82592c7da49383d1b1650e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 2 Dec 2021 00:40:25 -0500 Subject: [PATCH 90/99] nit: add docstring, constant, additional max_size shrinkage --- modmail/utils/pagination.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index 877bc2d1..3625bdea 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -31,11 +31,14 @@ JUMP_LAST_LABEL = " \u276f\u276f " # >> STOP_PAGINATE_EMOJI = "\u274c" # [:x:] This is an emoji, which is treated differently from the above -logger: ModmailLogger = logging.getLogger(__name__) +NO_EMBED_FOOTER_BUMP = 15 _AUTOGENERATE = object() +logger: ModmailLogger = logging.getLogger(__name__) + + class ButtonPaginator(ui.View, DpyPaginator): """ A class that helps in paginating long messages/embeds, which can be interacted via discord buttons. @@ -78,6 +81,8 @@ def __init__( If source message is provided and only_users is NOT provided, the paginator will respond to the author of the source message. To override this, pass an empty list to `only_users`. + By default, an embed is created. However, a custom embed can + be passed, or None can be passed to not use an embed. """ self.index = 0 self._pages: List[str] = [] @@ -92,11 +97,11 @@ def __init__( # used if embed is None self.content = "" - if embed is None: + if self.embed is None: self.title = title # need to set the max_size down a few to be able to set a "footer" # page indicator is "page xx of xx" - self.max_size -= 15 + self.max_size -= NO_EMBED_FOOTER_BUMP + len(self.title or "") if self.title is not None: self.max_size -= len(title) if footer_text is not None: From 81b5692211c4872483f0f29eacb1fda8125cbeda Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 14 Dec 2021 23:29:14 -0500 Subject: [PATCH 91/99] chore: add bool support for paginator embed generation --- modmail/utils/pagination.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index 3625bdea..8207ec14 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -62,7 +62,7 @@ def __init__( contents: Union[List[str], str], /, source_message: Optional[discord.Message] = None, - embed: Optional[Embed] = _AUTOGENERATE, + embed: Union[Embed, bool, None] = _AUTOGENERATE, timeout: float = 180, *, footer_text: str = None, @@ -90,9 +90,11 @@ def __init__( self.suffix = suffix self.max_size = max_size self.linesep = linesep - if embed is _AUTOGENERATE: + if embed is _AUTOGENERATE or embed is True: self.embed = Embed() else: + if embed is False: + embed = None self.embed = embed # used if embed is None From 09a33d4576818b62c4b276092c2fb8e5f91a7048 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 2 Jan 2022 01:02:25 -0500 Subject: [PATCH 92/99] chore: address reviews restructuring, using 'startswith', other updates --- modmail/config.py | 48 ++++++++--------------------------- modmail/{utils => }/errors.py | 6 +++++ modmail/utils/converters.py | 21 +++++++++++++++ modmail/utils/pagination.py | 2 +- 4 files changed, 38 insertions(+), 39 deletions(-) rename modmail/{utils => }/errors.py (56%) create mode 100644 modmail/utils/converters.py diff --git a/modmail/config.py b/modmail/config.py index c2a22325..67491e12 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -16,7 +16,9 @@ import marshmallow.fields import marshmallow.validate +from modmail.errors import ConfigLoadError from modmail.log import ModmailLogger +from modmail.utils.converters import BetterPartialEmojiConverter _log_optional_deps = logging.getLogger("modmail.optional_dependencies") @@ -103,37 +105,11 @@ def _get_config_directory() -> pathlib.Path: ] -class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): - """ - Converts to a :class:`~discord.PartialEmoji`. - - This is done by extracting the animated flag, name and ID from the emoji. - """ - - async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: - """Convert a provided argument into an emoji object.""" - match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) - if match is not None: - groups = match.groupdict() - animated = bool(groups["animated"]) - emoji_id = int(groups["id"]) - name = groups["name"] - return discord.PartialEmoji(name=name, animated=animated, id=emoji_id) - - return discord.PartialEmoji(name=argument) - - def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" return defaultdict(_generate_default_dict) -class ConfigLoadError(Exception): - """Exception if the configuration failed to load from a local file.""" - - pass - - class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deserialize into a string.""" @@ -152,18 +128,16 @@ def convert(self, argument: typing.Union[str, int, discord.Colour]) -> discord.C if not isinstance(argument, str): argument = str(argument) - if argument[0] == "#": + if argument.startswith("#"): return self.parse_hex_number(argument[1:]) - if argument[0:2] == "0x": + if argument.startswith("0x"): rest = argument[2:] # Legacy backwards compatible syntax - if rest.startswith("#"): - return self.parse_hex_number(rest[1:]) return self.parse_hex_number(rest) arg = argument.lower() - if arg[0:3] == "rgb": + if arg.startswith("rgb"): return self.parse_rgb(arg) arg = arg.replace(" ", "_") @@ -374,7 +348,7 @@ class DeveloperConfig: @log_level.validator def _log_level_validator(self, a: attr.Attribute, value: int) -> None: """Validate that log_level is within 0 to 50.""" - if value not in range(0, 50 + 1): + if value not in range(51): raise ValueError("log_level must be an integer within 0 to 50, inclusive.") @@ -556,7 +530,7 @@ def load_env( if not existing_config_dict: existing_config_dict = defaultdict(_generate_default_dict) - new_config_dict = attr.asdict( + return attr.asdict( _build_class( BaseConfig, dotenv_file=env_file, @@ -565,8 +539,6 @@ def load_env( ) ) - return new_config_dict - def load_toml(path: os.PathLike = None) -> defaultdict: """ @@ -577,7 +549,7 @@ def load_toml(path: os.PathLike = None) -> defaultdict: logger.trace(f"function `load_toml` called to load toml file {path}") if path is None: - path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml"),) + path = CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml") else: # fully resolve path path = pathlib.Path(path) @@ -606,7 +578,7 @@ def load_yaml(path: os.PathLike) -> dict: """ logger.trace(f"function `load_yaml` called to load yaml file {path}") if path is None: - path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml"),) + path = CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml") else: path = pathlib.Path(path) @@ -620,7 +592,7 @@ def load_yaml(path: os.PathLike) -> dict: ("The provided yaml config file is not a regular file.", path.is_file(), False), ] - errors = list() + errors = [] for text, check, stop_checking in states: if check: continue diff --git a/modmail/utils/errors.py b/modmail/errors.py similarity index 56% rename from modmail/utils/errors.py rename to modmail/errors.py index 69a2fda7..1f314157 100644 --- a/modmail/utils/errors.py +++ b/modmail/errors.py @@ -8,3 +8,9 @@ class InvalidArgumentError(Exception): """Improper argument.""" pass + + +class ConfigLoadError(Exception): + """Exception if the configuration failed to load from a local file.""" + + pass diff --git a/modmail/utils/converters.py b/modmail/utils/converters.py new file mode 100644 index 00000000..bd748354 --- /dev/null +++ b/modmail/utils/converters.py @@ -0,0 +1,21 @@ +import discord + + +class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): + """ + Converts to a :class:`~discord.PartialEmoji`. + + This is done by extracting the animated flag, name and ID from the emoji. + """ + + async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: + """Convert a provided argument into an emoji object.""" + match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) + if match is not None: + groups = match.groupdict() + animated = bool(groups["animated"]) + emoji_id = int(groups["id"]) + name = groups["name"] + return discord.PartialEmoji(name=name, animated=animated, id=emoji_id) + + return discord.PartialEmoji(name=argument) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index e0f633a4..18ae894f 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -13,7 +13,7 @@ from discord.embeds import Embed, EmbedProxy from discord.ext.commands import Paginator as DpyPaginator -from modmail.utils.errors import InvalidArgumentError, MissingAttributeError +from modmail.errors import InvalidArgumentError, MissingAttributeError if TYPE_CHECKING: From 40877671bc789775236180cc5b7d774893847079 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 21:33:07 -0400 Subject: [PATCH 93/99] remove dependency on environs --- .pre-commit-config.yaml | 1 - modmail/__init__.py | 14 +++++++++----- modmail/log.py | 13 ++++++++++++- poetry.lock | 24 +----------------------- pyproject.toml | 1 - requirements.txt | 1 - 6 files changed, 22 insertions(+), 32 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6885c397..4f0f0eb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,6 @@ repos: - coloredlogs - desert - discord.py - - environs - marshmallow - python-dotenv - pyyaml diff --git a/modmail/__init__.py b/modmail/__init__.py index 715badfd..1d3a153b 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -5,13 +5,17 @@ from pathlib import Path import coloredlogs -import environs -from modmail.log import ModmailLogger +from modmail.log import ModmailLogger, get_log_level_from_name -env = environs.Env() -env.read_env(".env", recurse=False) +try: + import dotenv +except ModuleNotFoundError: + pass +else: + dotenv.load_dotenv(".env") + # On windows aiodns's asyncio support relies on APIs like add_reader (which aiodns uses) # are not guaranteed to be available, and in particular are not available when using the # ProactorEventLoop on Windows, this method is only supported with Windows SelectorEventLoop @@ -28,7 +32,7 @@ # this logging level is set to logging.TRACE because if it is not set to the lowest level, # the child level will be limited to the lowest level this is set to. -ROOT_LOG_LEVEL = env.log_level("MODMAIL_LOG_LEVEL", logging.TRACE) +ROOT_LOG_LEVEL = get_log_level_from_name(os.environ.get("MODMAIL_LOG_LEVEL", logging.TRACE)) FMT = "%(asctime)s %(levelname)10s %(name)15s - [%(lineno)5d]: %(message)s" DATEFMT = "%Y/%m/%d %H:%M:%S" diff --git a/modmail/log.py b/modmail/log.py index 65afd23f..85754943 100644 --- a/modmail/log.py +++ b/modmail/log.py @@ -1,5 +1,16 @@ import logging -from typing import Any +from typing import Any, Union + + +def get_log_level_from_name(name: Union[str, int]) -> int: + """Find the logging level given the provided name.""" + if isinstance(name, int): + return name + name = name.upper() + value = getattr(logging, name, "") + if not isinstance(value, int): + raise TypeError("name must be an existing logging level.") + return value class ModmailLogger(logging.Logger): diff --git a/poetry.lock b/poetry.lock index 282af7b3..fb17273e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -311,24 +311,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "environs" -version = "9.3.5" -description = "simplified environment variable parsing" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -marshmallow = ">=3.0.0" -python-dotenv = "*" - -[package.extras] -dev = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url", "flake8 (==4.0.1)", "flake8-bugbear (==21.9.2)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)", "tox"] -django = ["dj-database-url", "dj-email-url", "django-cache-url"] -lint = ["flake8 (==4.0.1)", "flake8-bugbear (==21.9.2)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)"] -tests = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url"] - [[package]] name = "execnet" version = "1.9.0" @@ -1214,7 +1196,7 @@ yaml = ["PyYAML"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "eacd0d76072c6df35c5193d3ad9cb6d99848016175ee452359b6a4402f2f50b9" +content-hash = "0f700351dce1358d3f77c905e29618c3f8416e32cafe6c2c0158e6b5bd6799b9" [metadata.files] aiodns = [ @@ -1507,10 +1489,6 @@ distlib = [ docopt = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, ] -environs = [ - {file = "environs-9.3.5-py2.py3-none-any.whl", hash = "sha256:f5a3ca00afd32a82e098597c7dc8899bea5ed451d31055e518e8c7681cdf1c2b"}, - {file = "environs-9.3.5.tar.gz", hash = "sha256:47128992b854cf7e279e62127100767d765a1ead8d22f355e63c00661550131b"}, -] execnet = [ {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, diff --git a/pyproject.toml b/pyproject.toml index c03392be..d6e2f77d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ coloredlogs = "^15.0" atoml = "^1.0.3" attrs = "^21.2.0" desert = "^2020.11.18" -environs = "~=9.3.3" marshmallow = "~=3.13.0" python-dotenv = "^0.19.0" PyYAML = { version = "^5.4.1", optional = true } diff --git a/requirements.txt b/requirements.txt index 68067e4d..6205a0ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,6 @@ colorama==0.4.4 ; python_version >= "2.7" and python_version != "3.0" and python coloredlogs==15.0.1 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" desert==2020.11.18 ; python_version >= "3.6" discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_full_version >= "3.8.0" -environs==9.3.5 ; python_version >= "3.6" humanfriendly==10.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" idna==3.2 ; python_version >= "3.5" marshmallow-enum==1.5.1 From 5911ff845f316925485c1578d47213e88b42a10b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 21:54:57 -0400 Subject: [PATCH 94/99] fix: pin discord.py --- poetry.lock | 5 ++--- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index fb17273e..ec766640 100644 --- a/poetry.lock +++ b/poetry.lock @@ -293,8 +293,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] [package.source] type = "url" -url = "https://github.com/Rapptz/discord.py/archive/master.zip" - +url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip" [[package]] name = "distlib" version = "0.3.2" @@ -1196,7 +1195,7 @@ yaml = ["PyYAML"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "0f700351dce1358d3f77c905e29618c3f8416e32cafe6c2c0158e6b5bd6799b9" +content-hash = "c06b8fb70a86923177f0c1e3cf3c4be3bea709eac95482a62be9b5e3cf7d4a44" [metadata.files] aiodns = [ diff --git a/pyproject.toml b/pyproject.toml index d6e2f77d..5586ed7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ aiohttp = { extras = ["speedups"], version = "^3.7.4" } arrow = "^1.1.1" colorama = "^0.4.3" coloredlogs = "^15.0" -"discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/master.zip" } +"discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip" } atoml = "^1.0.3" attrs = "^21.2.0" desert = "^2020.11.18" diff --git a/requirements.txt b/requirements.txt index 6205a0ba..fc3854a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ chardet==4.0.0 ; python_version >= "2.7" and python_version != "3.0" and python_ colorama==0.4.4 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" coloredlogs==15.0.1 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" desert==2020.11.18 ; python_version >= "3.6" -discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_full_version >= "3.8.0" +discord.py @ https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip ; python_full_version >= "3.8.0" humanfriendly==10.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" idna==3.2 ; python_version >= "3.5" marshmallow-enum==1.5.1 From 768277e8dd01383bd18cef3a12479ba7fccac384 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 22:01:41 -0400 Subject: [PATCH 95/99] update black --- poetry.lock | 96 ++++++++++++++++---------------------------------- pyproject.toml | 2 +- 2 files changed, 32 insertions(+), 66 deletions(-) diff --git a/poetry.lock b/poetry.lock index ec766640..1119dd5d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -105,29 +105,24 @@ testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3 [[package]] name = "black" -version = "21.9b0" +version = "22.3.0" description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.6.2" [package.dependencies] -click = ">=7.1.2" +click = ">=8.0.0" mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0,<1" +pathspec = ">=0.9.0" platformdirs = ">=2" -regex = ">=2020.1.8" -tomli = ">=0.2.6,<2.0.0" -typing-extensions = [ - {version = ">=3.10.0.0", markers = "python_version < \"3.10\""}, - {version = "!=3.10.0.1", markers = "python_version >= \"3.10\""}, -] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] +d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -python2 = ["typed-ast (>=1.4.2)"] uvloop = ["uvloop (>=0.15.2)"] [[package]] @@ -294,6 +289,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] [package.source] type = "url" url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip" + [[package]] name = "distlib" version = "0.3.2" @@ -1009,14 +1005,6 @@ python-versions = ">=3.6" [package.dependencies] pyyaml = "*" -[[package]] -name = "regex" -version = "2021.8.28" -description = "Alternative regular expression module, to replace re." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "requests" version = "2.26.0" @@ -1195,7 +1183,7 @@ yaml = ["PyYAML"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "c06b8fb70a86923177f0c1e3cf3c4be3bea709eac95482a62be9b5e3cf7d4a44" +content-hash = "4eaebb69142186eb444698004183844720ad7436976eb465ce0b45310f9f4c27" [metadata.files] aiodns = [ @@ -1270,8 +1258,29 @@ attrs = [ {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, ] black = [ - {file = "black-21.9b0-py3-none-any.whl", hash = "sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115"}, - {file = "black-21.9b0.tar.gz", hash = "sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, ] brotlipy = [ {file = "brotlipy-0.7.0-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2"}, @@ -1932,49 +1941,6 @@ pyyaml-env-tag = [ {file = "pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069"}, {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, ] -regex = [ - {file = "regex-2021.8.28-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d05ad5367c90814099000442b2125535e9d77581855b9bee8780f1b41f2b1a2"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3bf1bc02bc421047bfec3343729c4bbbea42605bcfd6d6bfe2c07ade8b12d2a"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f6a808044faae658f546dd5f525e921de9fa409de7a5570865467f03a626fc0"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a617593aeacc7a691cc4af4a4410031654f2909053bd8c8e7db837f179a630eb"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79aef6b5cd41feff359acaf98e040844613ff5298d0d19c455b3d9ae0bc8c35a"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0fc1f8f06977c2d4f5e3d3f0d4a08089be783973fc6b6e278bde01f0544ff308"}, - {file = "regex-2021.8.28-cp310-cp310-win32.whl", hash = "sha256:6eebf512aa90751d5ef6a7c2ac9d60113f32e86e5687326a50d7686e309f66ed"}, - {file = "regex-2021.8.28-cp310-cp310-win_amd64.whl", hash = "sha256:ac88856a8cbccfc14f1b2d0b829af354cc1743cb375e7f04251ae73b2af6adf8"}, - {file = "regex-2021.8.28-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c206587c83e795d417ed3adc8453a791f6d36b67c81416676cad053b4104152c"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8690ed94481f219a7a967c118abaf71ccc440f69acd583cab721b90eeedb77c"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:328a1fad67445550b982caa2a2a850da5989fd6595e858f02d04636e7f8b0b13"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c7cb4c512d2d3b0870e00fbbac2f291d4b4bf2634d59a31176a87afe2777c6f0"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66256b6391c057305e5ae9209941ef63c33a476b73772ca967d4a2df70520ec1"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8e44769068d33e0ea6ccdf4b84d80c5afffe5207aa4d1881a629cf0ef3ec398f"}, - {file = "regex-2021.8.28-cp36-cp36m-win32.whl", hash = "sha256:08d74bfaa4c7731b8dac0a992c63673a2782758f7cfad34cf9c1b9184f911354"}, - {file = "regex-2021.8.28-cp36-cp36m-win_amd64.whl", hash = "sha256:abb48494d88e8a82601af905143e0de838c776c1241d92021e9256d5515b3645"}, - {file = "regex-2021.8.28-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b4c220a1fe0d2c622493b0a1fd48f8f991998fb447d3cd368033a4b86cf1127a"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a332404baa6665b54e5d283b4262f41f2103c255897084ec8f5487ce7b9e8e"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c61dcc1cf9fd165127a2853e2c31eb4fb961a4f26b394ac9fe5669c7a6592892"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ee329d0387b5b41a5dddbb6243a21cb7896587a651bebb957e2d2bb8b63c0791"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f60667673ff9c249709160529ab39667d1ae9fd38634e006bec95611f632e759"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b844fb09bd9936ed158ff9df0ab601e2045b316b17aa8b931857365ea8586906"}, - {file = "regex-2021.8.28-cp37-cp37m-win32.whl", hash = "sha256:4cde065ab33bcaab774d84096fae266d9301d1a2f5519d7bd58fc55274afbf7a"}, - {file = "regex-2021.8.28-cp37-cp37m-win_amd64.whl", hash = "sha256:1413b5022ed6ac0d504ba425ef02549a57d0f4276de58e3ab7e82437892704fc"}, - {file = "regex-2021.8.28-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ed4b50355b066796dacdd1cf538f2ce57275d001838f9b132fab80b75e8c84dd"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28fc475f560d8f67cc8767b94db4c9440210f6958495aeae70fac8faec631797"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc178caebd0f338d57ae445ef8e9b737ddf8fbc3ea187603f65aec5b041248f"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999ad08220467b6ad4bd3dd34e65329dd5d0df9b31e47106105e407954965256"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:808ee5834e06f57978da3e003ad9d6292de69d2bf6263662a1a8ae30788e080b"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d5111d4c843d80202e62b4fdbb4920db1dcee4f9366d6b03294f45ed7b18b42e"}, - {file = "regex-2021.8.28-cp38-cp38-win32.whl", hash = "sha256:473858730ef6d6ff7f7d5f19452184cd0caa062a20047f6d6f3e135a4648865d"}, - {file = "regex-2021.8.28-cp38-cp38-win_amd64.whl", hash = "sha256:31a99a4796bf5aefc8351e98507b09e1b09115574f7c9dbb9cf2111f7220d2e2"}, - {file = "regex-2021.8.28-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:04f6b9749e335bb0d2f68c707f23bb1773c3fb6ecd10edf0f04df12a8920d468"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b006628fe43aa69259ec04ca258d88ed19b64791693df59c422b607b6ece8bb"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:121f4b3185feaade3f85f70294aef3f777199e9b5c0c0245c774ae884b110a2d"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a577a21de2ef8059b58f79ff76a4da81c45a75fe0bfb09bc8b7bb4293fa18983"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1743345e30917e8c574f273f51679c294effba6ad372db1967852f12c76759d8"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e1e8406b895aba6caa63d9fd1b6b1700d7e4825f78ccb1e5260551d168db38ed"}, - {file = "regex-2021.8.28-cp39-cp39-win32.whl", hash = "sha256:ed283ab3a01d8b53de3a05bfdf4473ae24e43caee7dcb5584e86f3f3e5ab4374"}, - {file = "regex-2021.8.28-cp39-cp39-win_amd64.whl", hash = "sha256:610b690b406653c84b7cb6091facb3033500ee81089867ee7d59e675f9ca2b73"}, - {file = "regex-2021.8.28.tar.gz", hash = "sha256:f585cbbeecb35f35609edccb95efd95a3e35824cd7752b586503f7e6087303f1"}, -] requests = [ {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, diff --git a/pyproject.toml b/pyproject.toml index 5586ed7e..d7fdc563 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ yaml = ["pyyaml"] pre-commit = "~=2.1" taskipy = "^1.6.0" # linting -black = "^21.7b0" +black = "^22.3.0" flake8 = "~=3.8" flake8-annotations = "~=2.3" flake8-bugbear = "~=20.1" From 62e82fa35c52e52852b43198ed135f77a53ae62b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 22:03:16 -0400 Subject: [PATCH 96/99] update precommit hooks --- .pre-commit-config.yaml | 8 ++++---- modmail/__init__.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f0f0eb4..ba9267d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: - pyyaml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.2.0 hooks: - id: check-case-conflict - id: check-added-large-files @@ -65,19 +65,19 @@ repos: - id: python-use-type-annotations - repo: https://github.com/PyCQA/isort - rev: 5.9.3 + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/asottile/blacken-docs - rev: v1.11.0 + rev: v1.12.1 hooks: - id: blacken-docs additional_dependencies: - black - repo: https://github.com/psf/black - rev: 21.8b0 + rev: 22.3.0 hooks: - id: black language_version: python3 diff --git a/modmail/__init__.py b/modmail/__init__.py index 1d3a153b..708274df 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -28,7 +28,7 @@ logging.addLevelName(logging.NOTICE, "NOTICE") -LOG_FILE_SIZE = 8 * (2 ** 10) ** 2 # 8MB, discord upload limit +LOG_FILE_SIZE = 8 * (2**10) ** 2 # 8MB, discord upload limit # this logging level is set to logging.TRACE because if it is not set to the lowest level, # the child level will be limited to the lowest level this is set to. From 4644d760c5f185ee8899534f9214d91aca6ab365 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 22:18:36 -0400 Subject: [PATCH 97/99] add configuration system to docs --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 11a81a73..ebe927bd 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added Dispatcher system, although it is not hooked into important features like thread creation yet. (#71) - Officially support python 3.10 (#119) - Officially support windows and macos (#121) +- Completely rewrote configuration system (#75) ### Changed From 0c4e1c22c3584fcd9c9e3809cbd38a7756fa1972 Mon Sep 17 00:00:00 2001 From: arl Date: Wed, 20 Apr 2022 23:12:38 -0400 Subject: [PATCH 98/99] Add Threads/Tickets. (#53) * update discord py * feat: monkey-patch embeds to give more init param options * feat: add dmrelay cog for relaying dms to channel some values are hardcoded until the configuration system is reimplemented * feat: Add close command and enhance dmrelay support via webhooks * fix: add missing config value to default config temporary fix until the configuration system is rewritten * chore: use correct types, pass thread, archive after message. * chore: remove support for webhooks Using webhooks to send messages do not add much if any benefit, as a webhook with the member username and pfp is a bit harder to tell apart from staff at first glance. This is also the only benefit that using webhooks give, for the multiple disadventages they give, as now the bot has to manage a webhook too. We may support webhooks in the future as an official plugin, but for now we are focusing on the core functionality of the bot. * chore: rename DmRelay to Tickets * chore: move thread decorator to threads.py * threads: add ticket model * chore: update discordpy * chore: [temp fix] add members intent * chore: don't have any hardcoded config * threads: rewrite thread decorator Thread decorator now also checks if it is a modmail thread or not, to ensure ticket only commands aren't enabled on some other thread. * chore[DMs]: add helper method to check if a user can be dmed * chore[dependencies]: run poetry update * minor: rename utils.user to utils.users * feat: add ticket system to relay messages from dms to server, and back IT LIVES!! As of this commit, the bot can now function as a modmail bot, albiet it is not fully implemented, and is somewhat buggy, but the basic implementation is there. * minor: add empty test file for threads/tickets * chore: add required variable to .env.template * minor: get rid of unused parameter * nit: fix docstring in not-yet-written tests * changes: document threads existence, and breaking env var * nit: fix dms blocked alert message * minor: make open a top level command named contact * minor: add logging statements to threads.py * minor: use TYPE_CHECKING * threads: add on_typing relay * tools: run poetry update * threads: add typing event relay * threads: add embed format stub methods * breaking: refactor threads * threads: move Ticket objects to be stored on the bot obj * chore: make the embed creator an object This is for when configuration will need to be passed to the embed creator. Most embed options will be customisable, and it is easier to pass a configuration object to the embed creator once than it is to pass configuration parameters to every single method. * threads: implement more embeds, add buggy edit command * fix: send error message when thread can't be created * chore: update docstring of contact command * fix: add delete command, make edit command work * fix: channel name now matches recipient * threads: add user-side close embed * chore: update discord.py and use Member.display_avatar * add closing on thread archive events closes tickets on thread archive events also implements a lock for adding and removing threads * fix: use audit logs so its possible to detect the user who archived the thread * minor: make audit log optional at expense of not knowing who closed a thread * nit: remove duped author info from embeds * scrap thread embed creator was too complicated, didn't serve a good use at the moment, could possibly make a return in the future but very unlikely * minor: type the message dict return type properly * chore: show ids on server relay but not on dm relay * minor changes to support relaying embeds between servers * chore: get rid of useless message ID * fix: patch getitem method of messagedict * minor: split relay into two methods * chore: patch message dict to only allow discord.Message objects * feat: listen to edit and deletion dm events and relay them to guild * fix: edit works even if the last message was deleted and delete can delete multiple messages * nit: save users to dm_channel to user mapping if they don't exist * fix: handle stickers and developer debug command. * minor: use built in discord timestamps with arrow * add better attachment support and thread to dm deletion relay * relay attachments from guild to dm * fix handling of stickers and attachments * fix: multiple tickets can't be opened with a user * fix: send thread open message on opens * minor: don't relay a thread close msg if no messages were sent * fix: allow mods to provide contents when closing a thread * fix: add missing type annotations * fix: message sent in server is now better looking * fix: rename thread debug command to prevent future conflicts * nit: change user_dms_closed message * Partially Revert 18d4f9389279ddd10d1596b3e2b3072373aad835 * chore: update comments and docs as requested * chore: relock the lockfile * nit: add missing period and use f-string * fix: different colours for received and sent message embeds * minor: move opened-at timestamp to a field * nit: don't specify None when using get() on a dict * make first wave of suggested changes I need to commit and push, will squash these after * make second wave of suggested changes I need to commit and push, will squash these after * fix: ignore all checkfailures from dm commands * fix: remove unused converters these converters may be useful, but they are not needed at this time * nit: better detail 100% error rate on check_can_dm_user * chore: unprivatize most thread methods * minor: privatize ticket dict * fix: remove additional files as part of 03ec70fe25bf16e9f74552b9879a4aef815416c2 * nit: rename thread test file * chore: make some requsted changes - rename kwargs 'raise_for_preexisting_ticket' to 'raise_for_preexisting' - don't use discord.ext.tasks for a simple bot.loop.create_task event - fix a few typos in commands - define and label the sets used for handling deleted messages * minor: use modmail.utils.time for timestamps * minor: fix edited messages not being relayed to guild * fix: method now returns instance of class instead of class other minor changes including annotations and comments * tests: add some tests for threads * tests: add tests for utils.thread.decorators * tests: add tests for the contact command * nit: remove some extra trailing commas and move typing indicator * tests: add general test for reply_to_user * fix: use async rather than sync method for fetching tickets * chore: rename variables, use named format strings, and touch up error ui * nit: use valid kwargs footer_text instead of footer * chore: standardize and dedupe code for edit and delete commands * chore: refactor fetch_ticket to return None if no ticket found * chore: rename variables, fix logic bug, make constant * fix: use correct logic a few if statements were checking a variable is not None, they should be checking the variable is None Co-authored-by: Shivansh-007 --- docs/changelog.md | 9 +- modmail/bot.py | 5 +- modmail/config.py | 22 + modmail/default_config.toml | 4 + modmail/default_config.yaml | 3 + modmail/extensions/threads.py | 1137 +++++++++++++++++ modmail/utils/threads/__init__.py | 3 + modmail/utils/threads/decorators.py | 24 + modmail/utils/threads/errors.py | 16 + modmail/utils/threads/models.py | 83 ++ modmail/utils/users.py | 18 + tests/modmail/extensions/test_threads.py | 349 +++++ .../modmail/utils/threads/test_decorators.py | 40 + tests/modmail/utils/threads/test_models.py | 106 ++ tox.ini | 1 + 15 files changed, 1818 insertions(+), 2 deletions(-) create mode 100644 modmail/extensions/threads.py create mode 100644 modmail/utils/threads/__init__.py create mode 100644 modmail/utils/threads/decorators.py create mode 100644 modmail/utils/threads/errors.py create mode 100644 modmail/utils/threads/models.py create mode 100644 modmail/utils/users.py create mode 100644 tests/modmail/extensions/test_threads.py create mode 100644 tests/modmail/utils/threads/test_decorators.py create mode 100644 tests/modmail/utils/threads/test_models.py diff --git a/docs/changelog.md b/docs/changelog.md index ebe927bd..37033831 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,8 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Added +### Breaking +- Bot now requires a `RELAY_CHANNEL_ID` configuration variable. (#53) + - This is where tickets with users will be relayed. + - At a later point in time, this will be additionally included in a configuration command. +### Added +- Threads system (#53) + - Messages can now be relayed between a user and a server. + - NOTE: There is not a database yet, so none of these messages are stored. - Added Dispatcher system, although it is not hooked into important features like thread creation yet. (#71) - Officially support python 3.10 (#119) - Officially support windows and macos (#121) diff --git a/modmail/bot.py b/modmail/bot.py index f377019b..ed582f45 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -16,6 +16,7 @@ from modmail.log import ModmailLogger from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions from modmail.utils.plugins import PLUGINS, walk_plugins +from modmail.utils.threads import Ticket REQUIRED_INTENTS = Intents( @@ -38,9 +39,11 @@ class ModmailBot(commands.Bot): logger: ModmailLogger = logging.getLogger(__name__) dispatcher: Dispatcher + _tickets: t.Dict[int, Ticket] = dict() + def __init__(self, **kwargs): self.config = config() - self.start_time: t.Optional[arrow.Arrow] = None # arrow.utcnow() + self.start_time: arrow.Arrow = arrow.utcnow() self.http_session: t.Optional[aiohttp.ClientSession] = None self.dispatcher = Dispatcher() diff --git a/modmail/config.py b/modmail/config.py index 67491e12..33d975bf 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -6,6 +6,7 @@ import types import typing from collections import defaultdict +from typing import Optional import attr import desert @@ -383,6 +384,26 @@ class EmojiConfig: ) +@attr.mutable(slots=True) +class ThreadConfig: + """Thread configuration.""" + + thread_mention_role_id: Optional[int] = attr.ib( + default=0, + metadata={ + METADATA_TABLE: ConfigMetadata( + description="Role to mention on ticket creation.", + ) + }, + converter=int, + ) + relay_channel_id: Optional[int] = attr.ib( + default=0, + metadata={METADATA_TABLE: ConfigMetadata(description="Channel to use for creating tickets.")}, + converter=int, + ) + + @attr.s(auto_attribs=True, slots=True) class BaseConfig: """ @@ -406,6 +427,7 @@ class BaseConfig: }, ) emojis: EmojiConfig = EmojiConfig() + threads: ThreadConfig = ThreadConfig() # build configuration diff --git a/modmail/default_config.toml b/modmail/default_config.toml index 56f16779..153016fc 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -19,3 +19,7 @@ production = true [emojis] failure = ":x:" success = ":thumbsup:" + +[threads] +relay_channel_id = 0 +thread_mention_role_id = 0 diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index eb88ad23..883ca85e 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -14,3 +14,6 @@ dev: emojis: failure: ':x:' success: ':thumbsup:' +threads: + relay_channel_id: 0 + thread_mention_role_id: 0 diff --git a/modmail/extensions/threads.py b/modmail/extensions/threads.py new file mode 100644 index 00000000..679a3f7a --- /dev/null +++ b/modmail/extensions/threads.py @@ -0,0 +1,1137 @@ +import asyncio +import contextlib +import copy +import datetime +import inspect +import logging +from typing import TYPE_CHECKING, Dict, Generator, List, NoReturn, Optional, Set, Tuple, Union + +import arrow +import discord +from discord import Embed +from discord.ext import commands +from discord.ext.commands import Context +from discord.utils import escape_markdown + +from modmail.utils.cogs import ExtMetadata, ModmailCog +from modmail.utils.extensions import BOT_MODE, BotModes +from modmail.utils.threads import Ticket, is_modmail_thread +from modmail.utils.threads.errors import ThreadAlreadyExistsError, ThreadNotFoundError +from modmail.utils.time import TimeStampEnum, get_discord_formatted_timestamp +from modmail.utils.users import check_can_dm_user + + +if TYPE_CHECKING: # pragma: nocover + from modmail.bot import ModmailBot + from modmail.log import ModmailLogger + +EXT_METADATA = ExtMetadata() + +DEV_MODE_ENABLED = BOT_MODE & BotModes.DEVELOP + +BASE_JUMP_URL = "https://discord.com/channels" +DM_FAILURE_MESSAGE = ( + "**{user!s}** is not able to be DMed! This is because they have either blocked the bot, " + "or they are only accepting direct messages from friends.\n" + "It is also possible that they do not share a server with the bot." +) +ON_SUCCESS_EMOJI = "\u2705" # ✅ +ON_FAILURE_EMOJI = "\u274c" # :x: + +# This will be part of configuration later, so its stored in globals for now +FORWARD_USER_TYPING = False # Library bug prevents this from working right now +FORWARD_MODERATOR_TYPING = False + +# NOTE: Since discord removed `threads.archiver_id`, (it will always be `None` now), and the +# only way to get the user who archived the thread is to use the Audit logs. +# however, this permission is not required to have basic functionality. +# this permission +USE_AUDIT_LOGS = True + +NO_REPONSE_COLOUR = discord.Colour.red() +HAS_RESPONSE_COLOUR = discord.Colour.yellow() +CLOSED_COLOUR = discord.Colour.green() + +FORWARDED_DM_COLOR = discord.Colour.dark_teal() # messages received from dms +INTERNAL_REPLY_COLOR = discord.Colour.teal() # messages sent, shown in thread + +MAX_CACHED_MESSAGES_PER_THREAD = 10 + +IMAGE_EXTENSIONS = (".png", ".apng", ".gif", ".webm", "jpg", ".jpeg") + +logger: "ModmailLogger" = logging.getLogger(__name__) + + +class RepliedOrRecentMessageConverter(commands.Converter): + """ + Custom converter to return discord Message from within modmail threads. + + First attempts with the standard message converter, and upon failure, + will attempt to get the referenced message. + If that fails, will pop the ticket's recent messages. + """ + + def __init__(self, optional: bool = False, require_argument_empty: bool = False): + """ + Set up state. + + Optional + True - return tuple of message/errors + False - return tuple of message/None, raise errors + + require_argument_empty + True - requires passed argument is effectively None + False - doesn't do above check + """ + self.optional = optional + self.require_argument_empty = require_argument_empty + + async def convert(self, ctx: Context, argument: str) -> Tuple[Optional[discord.Message], Optional[str]]: + """Converting implementation. See class docstring for more information.""" + print(ctx.command.name) + + def or_raise(err: Exception) -> Union[Tuple[None, Exception], NoReturn]: + if self.optional: + return None, err + else: + raise err + + try: + return await commands.MessageConverter().convert(ctx, argument), None + except commands.CommandError: + pass + + if self.require_argument_empty and len(argument): + return or_raise(commands.CommandError("Provided argument is not empty.")) + + if ctx.message.reference is not None: + ref = ctx.message.reference + message = ref.resolved or await ctx.channel.fetch_message(ref.message_id) + else: + ticket: Ticket = await ctx.bot.get_cog("Threads").fetch_ticket(ctx.channel.id) + if ticket is None: + return or_raise(commands.CommandError("There's no message here to action on!")) + try: + message = ticket.last_sent_messages[-1] + except IndexError: + return or_raise(commands.CommandError("There's no message here to action on!")) + + # undo eating this parameter + # this means that the argument passed here will be passed to the next parameter + # but thankfully we can still get our result from here + ctx.view.undo() + + if message.author.id != ctx.bot.user.id: + logger.info("Edit command contained a reply, but was not to one of my messages so skipping.") + return or_raise(commands.CommandError("The replied message is not a message I can modify.")) + + return message, None + + +class TicketsCog(ModmailCog, name="Threads"): + """A cog for relaying direct messages.""" + + def __init__(self, bot: "ModmailBot"): + self.bot = bot + super().__init__(bot) + # validation for this configuration variable is be defered to fully implementing + # a new configuration system + self.relay_channel: Union[ + discord.TextChannel, discord.PartialMessageable + ] = self.bot.get_partial_messageable(self.bot.config.user.threads.relay_channel_id) + + self.dms_to_users: Dict[int, int] = dict() # key: dm_channel.id, value: user.id + + # message deletion events are messed up, so we have to use these sets + # to track if we deleted a message, and if we have already relayed it or not. + # these lists hold the ids of deleted messages that have + # been acted on before a on_msg_delete event is received + + self.dm_deleted_messages: Set[int] = set() # message.id of the bot's deleted messages in dms + self.thread_deleted_messages: Set[int] = set() # message.id of the bot's deleted messsages in thread + + self.thread_create_delete_lock = asyncio.Lock() + self.thread_create_lock = asyncio.Lock() + + self.use_audit_logs: bool = USE_AUDIT_LOGS + self.bot.loop.create_task(self.fetch_necessary_values()) + + async def init_relay_channel(self) -> None: + """Fetch the relay channel.""" + self.relay_channel = await self.bot.fetch_channel(self.bot.config.user.threads.relay_channel_id) + + async def fetch_necessary_values(self) -> None: + """Fetch the audit log permission.""" + self.relay_channel: discord.TextChannel = await self.bot.fetch_channel(self.relay_channel.id) + # a little bit of hackery because for some odd reason, the guild object is not always complete + self.relay_channel.guild = await self.bot.fetch_guild(self.relay_channel.guild.id) + me = await self.relay_channel.guild.fetch_member(self.bot.user.id) + self.use_audit_logs = USE_AUDIT_LOGS and me.guild_permissions.view_audit_log + logger.debug("Fetched relay channel and use_audit_log perms") + + def cog_unload(self) -> None: + """Cancel any tasks that may be running on unload.""" + super().cog_unload() + + async def add_ticket(self, ticket: Ticket, /) -> Ticket: + """Save a newly created ticket.""" + self.bot._tickets[ticket.recipient.id] = ticket + self.bot._tickets[ticket.thread.id] = ticket + return ticket + + async def fetch_ticket(self, id: int, /, raise_exception: bool = False) -> Optional[Ticket]: + """ + Fetch a ticket from the tickets dict. + + In the future this will be hooked into the database. + + By default, returns None if a ticket cannot be found. + However, if raise_exception is True, then this function will raise a ThreadNotFoundError + """ + # given that this is an async method, it is expected to yield somewhere + # this gives way to any waiting coroutines while here, temporarily + await asyncio.sleep(0) + try: + ticket = self.bot._tickets[id] + except KeyError: + if raise_exception: + raise ThreadNotFoundError(f"Could not find thread from id {id}.") + return None + else: + return ticket + + def get_user_from_dm_channel_id(self, id: int, /) -> int: + """Get a user id from a dm channel id. Raises a KeyError if user is not found.""" + return self.users_to_channels[id] + + # the reason we're checking for a user here rather than a member is because of future support for + # a designated server to handle threads and a server where the community resides, + # so it's possible that the user isn't in the server where this command is run. + @commands.command() + async def contact( + self, ctx: Context, recipient: Union[discord.User, discord.Member], *, reason: str = "" + ) -> None: + """ + Open a new ticket with a provided recipient. + + This will create a new ticket with the recipient, if a ticket does not already exist. + If a ticket already exists, a message will be sent in reply with a link to the existing ticket. + If the user is not able to be DMed, a message will be sent to the channel. + """ + if recipient.bot: + if recipient == self.bot.user: + await ctx.send("I'm perfect, you can't open a ticket with me.") + return + await ctx.send("You can't open a ticket with a bot.") + return + + try: + async with self.thread_create_lock: + ticket = await self.create_ticket( + ctx.message, + recipient=recipient, + raise_for_preexisting=True, + send_initial_message=False, + description=reason, + creator=ctx.message.author, + ) + except ThreadAlreadyExistsError: + thread = (await self.fetch_ticket(recipient.id)).thread + await ctx.send( + f"A thread already exists with {recipient.mention} ({recipient.id})." + f"You can find it here: <{BASE_JUMP_URL}/{thread.guild.id}/{thread.id}>", + allowed_mentions=discord.AllowedMentions(users=False), + ) + return + + if not await check_can_dm_user(recipient): + await ticket.thread.send(DM_FAILURE_MESSAGE.format(user=escape_markdown(str(recipient)))) + + async def create_ticket( + self, + initial_message: discord.Message, + /, + *, + recipient: discord.User = None, + raise_for_preexisting: bool = True, + send_initial_message: bool = True, + description: str = None, + creator: Union[discord.User, discord.Member] = None, + ) -> Ticket: + """ + Creates a bot ticket with a user. Also adds it to the tickets dict. + + One of recipient and initial_message must be provided. + If recipient is not provided, it will be determined from the initial_message. + + Parameters + ---------- + initial_message: discord.Message + + recipient: discord.User + + raise_for_preexisting: bool = True + Whether to check if there is an existing ticket for the user. + If there is an existing thread, this method will raise a ThreadAlreadyExistsError exception. + + send_initial_message: bool = True + Whether to relay the provided initial_message to the user. + + """ + recipient = recipient or initial_message.author + + # lock this next session, since we're checking if a thread already exists here + # we want to ensure that anything entering this section can get validated. + async with self.thread_create_delete_lock: + if recipient.id in self.bot._tickets.keys(): + if raise_for_preexisting: + raise ThreadAlreadyExistsError(recipient.id) + else: + return await self.fetch_ticket(recipient.id) + + thread_channel, thread_msg = await self._start_discord_thread( + initial_message, recipient, description=description, creator=creator + ) + ticket = Ticket( + recipient, + thread_channel, + has_sent_initial_message=send_initial_message, + log_message=thread_msg, + ) + # add the ticket as both the recipient and the thread ids so + # the tickets can be retrieved from both users or threads. + await self.add_ticket(ticket) + + # also save user dm channel id + if recipient.dm_channel is None: + await recipient.create_dm() + self.dms_to_users[recipient.dm_channel.id] = recipient.id + + return ticket + + async def _start_discord_thread( + self, + message: discord.Message, + recipient: discord.User = None, + *, + embed: discord.Embed = None, + description: str = None, + creator: Union[discord.User, discord.Member] = None, + **send_kwargs, + ) -> Tuple[discord.Thread, discord.Message]: + """ + Create a thread in discord off of a provided message. + + Sends an initial message, and returns the thread and the first message sent in the thread. + Any kwargs not defined in the method signature are forwarded to the discord.Messageable.send method + """ + await self.init_relay_channel() + + recipient = recipient or message.author + if send_kwargs.get("allowed_mentions") is None: + send_kwargs["allowed_mentions"] = discord.AllowedMentions( + everyone=False, users=False, roles=True, replied_user=False + ) + # TODO: !CONFIG add to configuration system. + if self.bot.config.user.threads.thread_mention_role_id is not None: + mention = f"<@&{self.bot.config.user.threads.thread_mention_role_id}>" + else: + mention = "@here" + + if description is None: + description = message.content + description = description.strip() + + if creator: + description += f"\nOpened by {creator!s}" + + embed = discord.Embed( + title=f"{discord.utils.escape_markdown(recipient.name,ignore_links=False)}" + f"#{recipient.discriminator} (`{recipient.id}`)", + description=f"{description}", + timestamp=datetime.datetime.now(), + color=NO_REPONSE_COLOUR, + ) + embed.add_field( + name="Opened since", + value=get_discord_formatted_timestamp(arrow.utcnow(), TimeStampEnum.RELATIVE_TIME), + ) + + relayed_msg = await self.relay_channel.send(content=mention, embed=embed, **send_kwargs) + try: + thread_channel = await relayed_msg.create_thread( + name=f"{recipient!s}".replace("#", "-"), + auto_archive_duration=relayed_msg.channel.default_auto_archive_duration, + ) + except discord.HTTPException as e: + if e.code != 50_035: + raise + # repeat the request but using the user id and discrim + # 50035 means that the user has some banned characters or phrases in their name + thread_channel = await relayed_msg.create_thread( + name=recipient.id, auto_archive_duration=relayed_msg.channel.default_auto_archive_duration + ) + + return thread_channel, relayed_msg + + async def resolve_mirror_message_for_manipulation( + self, + ctx: Context, + ticket: Ticket, + message: discord.Message, + ) -> Optional[Tuple[discord.Message, discord.Message]]: + """Find the corresponding dm message and raise any errors if the bot did not send that message.""" + try: + user_message = ticket.messages[message] + except KeyError: + try: + ticket.last_sent_messages.remove(message) + except ValueError: + pass + await ctx.send( + "Sorry, this is not a message that I can edit.", + reference=message.to_reference(fail_if_not_exists=False), + ) + + raise + + if user_message.author.id != self.bot.user.id: + raise commands.CommandError( + "DM message author is me. It seems like this was a message that you received." + ) + return user_message, message + + @contextlib.asynccontextmanager + async def handle_success(self, ctx: Context) -> Generator[None, None, None]: + """If any exceptions are thrown, a failure emoji is added and the exception is reraised.""" + try: + yield + except Exception: + await ctx.message.add_reaction(ON_FAILURE_EMOJI) + raise + else: + await ctx.message.add_reaction(ON_SUCCESS_EMOJI) + + @contextlib.asynccontextmanager + async def remove_on_success( + self, ticket: Ticket, *messages: discord.Message + ) -> Generator[None, None, None]: + """Remove provided messages from last sent messages if no errors.""" + yield + for message in messages: + try: + ticket.last_sent_messages.remove(message) + except ValueError: + pass + + async def relay_message_to_user( + self, ticket: Ticket, message: discord.Message, contents: str = None, *, delete: bool = True + ) -> discord.Message: + """Relay a message from guild to user.""" + if ticket.recipient.dm_channel is None: + # Note, this is the recommended way by discord.py to fetch a dm channel. + await ticket.recipient.create_dm() + + # thread -> dm + logger.debug( + "Relaying message id {message.id} by {message.author} " + "from thread {thread.id} to dm channel {dm_channel.id}.".format( + message=message, thread=ticket.thread, dm_channel=ticket.recipient.dm_channel + ) + ) + + embeds: List[Embed] = [ + Embed( + description=contents, + timestamp=message.created_at, + color=message.author.color, + author=message.author, + ) + ] + # make a reply if it was a reply + dm_reference_message = None + guild_reference_message = None + if message.reference is not None: + # don't error if the paired message on the server end was deleted + message.reference.fail_if_not_exists = False + guild_reference_message = message.reference + try: + dm_reference_message = ticket.messages[message.reference.message_id].to_reference( + fail_if_not_exists=False + ) + except KeyError: + pass + if len(message.attachments) > 0: + # don't delete when forwarding a message that has attachments, + # as that will invalidate the attachments + delete = False + for a in message.attachments: + # featuring the first image attachment as the embed image + if a.url.lower().endswith(IMAGE_EXTENSIONS): + if not embeds[0].image: + embeds[0].set_image(url=a.url) + continue + embeds[0].add_field(name=a.filename, value=a.proxy_url, inline=False) + + if len(message.stickers): + # since users can only send one sticker right now, we only care about the first one + sticker = await message.stickers[0].fetch() + # IF its possible, add the sticker url to the embed attachment + if ( + getattr(sticker, "format", discord.StickerFormatType.lottie) + == discord.StickerFormatType.lottie + ): + await message.channel.send("Nope! This sticker of a type which can't be shown to the user.") + return None + else: + if len(embeds[0].image) == 0: + embeds[0].set_image(url=sticker.url) + else: + embeds.append(Embed().set_image(url=sticker.url)) + + sent_message = await ticket.recipient.send(embeds=embeds, reference=dm_reference_message) + # deep copy embeds to not have an internal race condition. + embeds = copy.deepcopy(embeds) + + # also relay it in the thread channel + embeds[0].set_footer(text=f"User ID: {message.author.id}") + + embeds[0].colour = INTERNAL_REPLY_COLOR + guild_message = await ticket.thread.send(embeds=embeds, reference=guild_reference_message) + + if delete: + await message.delete() + + # add last sent message to the list + ticket.last_sent_messages.append(guild_message) + if len(ticket.last_sent_messages) > MAX_CACHED_MESSAGES_PER_THREAD: + ticket.last_sent_messages.pop(0) # keep list length to MAX_CACHED_MESSAGES_PER_THREAD + + # add messages to the dict + ticket.messages[guild_message] = sent_message + return sent_message + + async def relay_message_to_guild( + self, ticket: Ticket, message: discord.Message, contents: Optional[str] = None + ) -> discord.Message: + """Relay a message from user to guild.""" + if ticket.recipient.dm_channel is None: + await ticket.recipient.create_dm() + + # dm -> thread + logger.debug( + "Relaying message id {message.id} from dm channel {dm_channel.id}" + " with {message.author} to thread {thread.id}.".format( + message=message, thread=ticket.thread, dm_channel=ticket.recipient.dm_channel + ) + ) + # make a reply if it was a reply + guild_reference_message = None + if message.reference is not None: + # don't want to fail a reference + message.reference.fail_if_not_exists = False + try: + guild_reference_message = ticket.messages[message.reference.message_id].to_reference( + fail_if_not_exists=False + ) + except KeyError: + pass + + embed = Embed( + author=message.author, + timestamp=message.created_at, + footer_text=f"Message ID: {message.id}", + colour=FORWARDED_DM_COLOR, + ) + if contents is None: + embed.description = contents = str(f"{message.content}") + if len(message.attachments) > 0: + attachments = message.attachments + + for a in attachments: + if a.url.lower().endswith(IMAGE_EXTENSIONS): + if not embed.image: + embed.set_image(url=a.proxy_url) + continue + embed.add_field(name=a.filename, value=a.proxy_url, inline=False) + + sticker = None + if len(message.stickers) > 0: + # while stickers is a list, we only care about the first one because + # as of now, users cannot send more than one sticker in a message. + # if this changes, we will support it. + sticker = message.stickers[0] + sticker = await sticker.fetch() + # this can be one of two types of stickers, either a StandardSticker or a GuildSticker + # StandardStickers are not usable by bots, but GuildStickers are, if they're from + # the same guild + if getattr(sticker, "guild_id", False) == ticket.thread.guild.id and getattr( + sticker, "available", False + ): + pass + else: + + # IF its possible, add the sticker url to the embed attachment + if getattr( + sticker, "format", discord.StickerFormatType.lottie + ) != discord.StickerFormatType.lottie and not len(embed.image): + embed.set_image(url=sticker.url) + + # we can't use this sticker + if sticker.description is not None: + description = f"**Description:** {sticker.description.strip()}\n" + else: + description = "" + embed.add_field( + name="Received Sticker", + value=f"**Sticker**: {sticker.name}\n{description}\n[Click for file]({sticker.url})", + inline=False, + ) + sticker = None + + send_kwargs = dict() + if sticker is not None: + send_kwargs["stickers"] = [sticker] + + if message.activity is not None: + embed.add_field(name="Activity Sent", value="\u200b") + + if ( + 0 == len(embed.description) == len(embed.image) == len(embed.fields) + and send_kwargs.get("attachments") is None + and send_kwargs.get("stickers") is None + ): + logger.info( + f"SKIPPING relay of message id {message.id} from {message.author!s} due to nothing to relay." + ) + return None + + sent_message = await ticket.thread.send(embed=embed, reference=guild_reference_message, **send_kwargs) + + # add messages to the dict + ticket.messages[message] = sent_message + return sent_message + + async def mark_thread_responded(self, ticket: Ticket) -> bool: + """Mark thread as responded. Returns True upon success, and False if it was already marked.""" + if (log_embeds := ticket.log_message.embeds)[0].colour == NO_REPONSE_COLOUR: + log_embeds[0].colour = HAS_RESPONSE_COLOUR + await ticket.log_message.edit(embeds=log_embeds) + return True + return False + + @is_modmail_thread() + @commands.command(aliases=("r",)) + async def reply(self, ctx: Context, *, message: str = None) -> None: + """Send a reply to the user.""" + if message is None and not ctx.message.attachments and not ctx.message.stickers: + param = inspect.Parameter("message", 3) + raise commands.MissingRequiredArgument(param) + ticket = await self.fetch_ticket(ctx.channel.id) + if not ticket.has_sent_initial_message: + await ctx.trigger_typing() + logger.info( + "Sending initial message before replying on a thread " + "that was opened with the contact command." + ) + + await ticket.recipient.send( + embeds=[ + Embed( + title="Ticket Opened", + description="A moderator has opened this ticket to have a conversation with you.", + ) + ] + ) + ticket.has_sent_initial_message = True + + await asyncio.sleep(1) + + await self.relay_message_to_user(ticket, ctx.message, message) + + await self.mark_thread_responded(ticket) + + @is_modmail_thread() + @commands.command(aliases=("e", "ed")) + @commands.max_concurrency(1, commands.BucketType.channel, wait=True) + async def edit( + self, ctx: Context, message: RepliedOrRecentMessageConverter(optional=True) = None, *, content: str + ) -> None: + """ + Edit a message in the thread. + + If the message is a reply and no message is provided, the bot will attempt to use the replied message. + However, if the reply is *not* to the bot, no action is taken. + + If there is no reply or message provided, the bot will edit the last sent message. + """ + if message is None: + message, err = await RepliedOrRecentMessageConverter().convert(ctx, "") + else: + message, err = message + + if err is not None: + raise err + + ticket = await self.fetch_ticket(ctx.channel.id) + + # process and get proper message + messages = await self.resolve_mirror_message_for_manipulation( + ctx, + ticket, + message, + ) + + if messages is None: + return + + user_message, message = messages + + async with self.handle_success(ctx): + # edit user message + embed = user_message.embeds[0] + embed.description = content + await user_message.edit(embed=embed) + + # edit guild message + embed = message.embeds[0] + embed.description = content + await message.edit(embed=embed) + + @is_modmail_thread() + @commands.command(aliases=("d", "del")) + @commands.max_concurrency(1, commands.BucketType.channel, wait=True) + async def delete( + self, + ctx: Context, + message: RepliedOrRecentMessageConverter(require_argument_empty=True) = None, + ) -> None: + """ + Delete a message in the thread. + + If the message is a reply and no message is provided, the bot will attempt to use the replied message. + However, if the reply is *not* to the bot, no action is taken. + + If there is no reply or message provided, the bot will delete the last sent message. + """ + if message is None: + message, err = await RepliedOrRecentMessageConverter().convert(ctx, "") + else: + message, err = message + + if err is not None: + raise err + + ticket = await self.fetch_ticket(ctx.channel.id) + + # process and get proper message + messages = await self.resolve_mirror_message_for_manipulation( + ctx, + ticket, + message, + ) + + if messages is None: + return + + dm_message, thread_message = messages + + async with self.handle_success(ctx): + async with self.remove_on_success(ticket, thread_message): + self.dm_deleted_messages.add(dm_message.id) + await dm_message.delete() + + self.thread_deleted_messages.add(thread_message.id) + await thread_message.delete() + + async def close_thread( + self, + ticket: Ticket, + closer: Optional[Union[discord.User, discord.Member]] = None, + time: Optional[datetime.datetime] = None, + notify_user: Optional[bool] = None, + automatically_archived: bool = False, + *, + contents: str = None, + keep_thread_closed: bool = False, + ) -> None: + """ + Close the current thread after `after` time from now. + + Note: This method destroys the Ticket object. + + If keep_thread_closed is True, this method will not send any messages in the thread, + since that would re-open the thread. + """ + if notify_user is None: + notify_user = bool(ticket.has_sent_initial_message or len(ticket.messages) > 0) + + if closer: + thread_close_embed = discord.Embed( + title="Thread Closed", + description=contents or f"{closer.mention} has closed this Modmail thread.", + timestamp=arrow.utcnow().datetime, + ) + else: + thread_close_embed = discord.Embed( + title="Thread Closed", + description=contents or "This thread has been closed.", + timestamp=arrow.utcnow().datetime, + ) + + async with self.thread_create_delete_lock: + # clean up variables + if not keep_thread_closed: + await ticket.thread.send(embed=thread_close_embed) + if notify_user: + # user may have dms closed + try: + await ticket.recipient.send(embed=thread_close_embed) + except discord.HTTPException: + logger.debug(f"{ticket.recipient} is unable to be DMed. Skipping.") + pass + + try: + del self.bot._tickets[ticket.thread.id] + del self.bot._tickets[ticket.recipient.id] + except KeyError: + logger.warning("Ticket not found in tickets dict when attempting removal.") + # ensure we get rid of the ticket messages, as this can be an extremely large dict + else: + # remove the user's dm channel from the dict + try: + del self.dms_to_users[ticket.recipient.dm_channel.id] + except KeyError: + # not a problem if the user is already removed + pass + + del ticket.messages + + if (log_embeds := ticket.log_message.embeds)[0].colour != CLOSED_COLOUR: + log_embeds[0].colour = CLOSED_COLOUR + await ticket.log_message.edit(embeds=log_embeds) + + await ticket.thread.edit(archived=True, locked=False) + + if not closer: + logger.debug(f"{closer!s} has closed thread {ticket.thread!s}.") + else: + logger.debug(f"{ticket.thread!s} has been closed. A user was not provided.") + + @is_modmail_thread() + @commands.group(invoke_without_command=True) + async def close(self, ctx: Context, *, contents: str = None) -> None: + """Close the current thread after `after` time from now.""" + # TODO: Implement after duration + ticket = await self.fetch_ticket(ctx.channel.id) + if ticket is None: + await ctx.send("Error: this thread is not in the list of tickets.") + return + + await self.close_thread(ticket, ctx.author, contents=contents) + + @ModmailCog.listener(name="on_message") + async def on_dm_message(self, message: discord.Message) -> None: + """Relay all dms to a thread channel.""" + author = message.author + + if author.id == self.bot.user.id: + return + + if message.guild: + return + + ticket = await self.fetch_ticket(author.id) + if ticket is None: + # Thread doesn't exist, so create one. + async with self.thread_create_lock: + try: + ticket = await self.create_ticket(message, raise_for_preexisting=True) + except ThreadAlreadyExistsError: + # the thread already exists, so we still need to relay the message + # thankfully a keyerror should NOT happen now + ticket = await self.fetch_ticket(author.id) + msg = await self.relay_message_to_guild(ticket, message) + else: + msg = await self.relay_message_to_guild(ticket, message) + if msg is None: + return + await message.channel.send( + embeds=[ + Embed( + title="Ticket Opened", + description=f"Thanks for dming {self.bot.user.name}! " + "A member of our staff will be with you shortly!", + timestamp=message.created_at, + ) + ] + ) + else: + msg = await self.relay_message_to_guild(ticket, message) + if msg is None: + return + + await message.add_reaction(ON_SUCCESS_EMOJI) + + @ModmailCog.listener(name="on_raw_message_edit") + async def on_dm_message_edit(self, payload: discord.RawMessageUpdateEvent) -> None: + """ + Receive a dm message edit, and edit the message in the channel. + + In the future, this will be expanded to use a modified paginator. + """ + if payload.guild_id is not None: + return + + if payload.data["author"]["id"] == self.bot.user.id: + return + + logger.trace( + f'User ID {payload.data["author"]["id"]} has edited a message ' + f"in their dms with id {payload.message_id}" + ) + ticket = await self.fetch_ticket(int(payload.data["author"]["id"])) + if ticket is None: + logger.debug( + f"User {payload.data['author']['id']} edited a message in dms which " + "was related to a non-existant ticket." + ) + return + + guild_msg = ticket.messages[payload.message_id] + + new_embed = guild_msg.embeds[0] + + data = payload.data + if data.get("content") is not None: + new_embed.insert_field_at(0, name="Former contents", value=new_embed.description) + new_embed.description = data["content"] + + await guild_msg.edit(embed=new_embed) + + dm_channel = self.bot.get_partial_messageable(payload.channel_id, type=discord.DMChannel) + await dm_channel.send( + embed=discord.Embed( + "Successfully edited message.", + footer_text=f"Message ID: {payload.message_id}", + ), + reference=discord.MessageReference(message_id=payload.message_id, channel_id=payload.channel_id), + ) + + @ModmailCog.listener(name="on_raw_message_delete") + async def on_dm_message_delete(self, payload: discord.RawMessageDeleteEvent) -> None: + """ + Receive a dm message edit, and edit the message in the channel. + + In the future, this will be expanded to use a modified paginator. + """ + if payload.guild_id is not None: + return + + if payload.message_id in self.dm_deleted_messages: + logger.debug(f"Ignoring message deleted by self in {payload.channel_id}") + self.dm_deleted_messages.remove(payload.message_id) + return + + try: + # get the user id + author_id = self.dms_to_users[payload.channel_id] + except KeyError: + channel: discord.DMChannel = await self.bot.fetch_channel(payload.channel_id) + author_id = channel.recipient.id + # add user to dict + self.dms_to_users[payload.channel_id] = author_id + + logger.trace( + f"A message from {author_id} in dm channel {payload.channel_id} has " + f"been deleted with id {payload.message_id}." + ) + ticket = await self.fetch_ticket(author_id) + if ticket is None: + logger.debug( + f"User {author_id} edited a message in dms which was related to a non-existant ticket." + ) + return + + guild_msg = ticket.messages[payload.message_id] + + new_embed = guild_msg.embeds[0] + + new_embed.colour = discord.Colour.red() + new_embed.insert_field_at( + 0, name="Deleted", value=f"Deleted at {get_discord_formatted_timestamp(arrow.utcnow())}" + ) + await guild_msg.edit(embed=new_embed) + + dm_channel = self.bot.get_partial_messageable(payload.channel_id, type=discord.DMChannel) + await dm_channel.send(embed=discord.Embed("Successfully deleted message.")) + + @ModmailCog.listener(name="on_raw_message_delete") + async def on_thread_message_delete(self, payload: discord.RawMessageDeleteEvent) -> None: + """Automatically deletes a message in the dms if it was deleted on the moderator end.""" + if payload.guild_id is None: + return + + if payload.message_id in self.thread_deleted_messages: + self.thread_deleted_messages.remove(payload.message_id) + logger.debug( + f"SKIPPING mirror of deleted message {payload.message_id} since it was deleted via a command." + ) + return + + ticket = await self.fetch_ticket(payload.channel_id) + if ticket is None: + # not a valid ticket + return + + if ticket.thread.id != payload.channel_id: + logger.warn("I have no idea what happened. This is a good time to stop using the bot.") + return + + try: + dm_msg = ticket.messages[payload.message_id] + except KeyError: + # message was deleted as a command + return + + logger.info( + f"Relaying manual message deletion in {payload.channel_id} to {ticket.recipient.dm_channel}" + ) + self.dm_deleted_messages.add(dm_msg.id) + await dm_msg.delete() + + @ModmailCog.listener(name="on_typing") + async def on_typing( + self, + channel: discord.abc.Messageable, + user: Union[discord.User, discord.Member], + _: datetime.datetime, + ) -> None: + """Relay typing events to the thread channel.""" + if user.id == self.bot.user.id: + return + + # only work in dms or a thread channel + + if FORWARD_MODERATOR_TYPING and isinstance(channel, discord.Thread): + + ticket = await self.fetch_ticket(channel.id) + if ticket is None: + # Thread doesn't exist, so there's nowhere to relay the typing event. + return + logger.debug(f"Relaying typing event from {user!s} in {channel!s} to {ticket.recipient!s}.") + await ticket.recipient.trigger_typing() + + # ! Due to a library bug this tree will never be run + # it can be tracked here: https://github.com/Rapptz/discord.py/issues/7432 + elif FORWARD_USER_TYPING and isinstance(channel, discord.DMChannel): + ticket = await self.fetch_ticket(user.id) + if ticket is None: + # User doesn't have a ticket, so nowhere to relay the event. + return + else: + logger.debug(f"Relaying typing event from {user!s} in {channel!s} to {ticket.recipient!s}.") + + await ticket.thread.trigger_typing() + + else: + return + + @ModmailCog.listener("on_thread_update") + async def on_thread_archive(self, before: discord.Thread, after: discord.Thread) -> None: + """ + Archives a thread after a preset time of it being automatically archived. + + This trigger only handles thread archiving. + """ + # we only care about edits that archive the thread + if before.archived != after.archived and not after.archived: + return + + # channel must have the parent of the relay channel + # while this should never change, I'm using `before` in case for some reason + # threads get the support to change their parent channel, which would be great. + if before.parent_id != self.relay_channel.id: + return + # ignore the bot closing threads + # NOTE: archiver_id is always gonna be None. + # HACK: Grab an item from the audit log to get this user. + archiver = None + automatically_archived = False + if self.use_audit_logs: + async for event in after.guild.audit_logs(limit=4, action=discord.AuditLogAction.thread_update): + if ( + event.target.id == after.id + and not getattr(event.before, "archived", None) + and getattr(event.after, "archived", None) + ): + archiver = event.user + break + + if archiver is None: + automatically_archived = True + elif self.bot.user == archiver: + logger.trace("Received a thread archive event which was caused by me. Skipping actions.") + return + + if archiver is None: + # check the last message id + now = arrow.utcnow().datetime + last_message_time = discord.Object(after.last_message_id).created_at + print() + if before.auto_archive_duration <= (now - last_message_time).total_seconds(): + # thread was automatically archived, probably. + automatically_archived = True + + # checks passed, closing the ticket + ticket = await self.fetch_ticket(after.id) + if ticket is None: + logger.debug( + "While closing a ticket, somehow checks passed but the thread did not exist... " + "This is likely due to missing audit log permissions." + ) + return + + await self.close_thread(ticket, archiver, automatically_archived=automatically_archived) + + @is_modmail_thread() + @commands.command(name="debug_thread", enabled=DEV_MODE_ENABLED) + async def debug(self, ctx: Context, attr: str = None) -> None: + """Debug command. Requires a message reference (reply).""" + ticket = await self.fetch_ticket(ctx.channel.id) + if ticket is None: + await ctx.send("no ticket found associated with this channel.") + return + dm_msg = ticket.messages[ctx.message.reference.message_id] + + if attr is None: + attribs: Dict[str] = {} + longest_len = 0 + for attr in dir(dm_msg): + if attr.startswith("_"): + continue + thing = getattr(dm_msg, attr) + if callable(thing): + continue + attribs[attr] = thing + if len(attr) > longest_len: + longest_len = len(attr) + + con = "" + longest_len += 2 + for k, v in attribs.items(): + con += f"{k.rjust(longest_len, ' ')}: {v!r}\n" + + else: + con = getattr(dm_msg, attr, "UNSET") + await ctx.send(f"```py\n{con}```") + + async def cog_command_error(self, ctx: Context, error: commands.CommandError) -> None: + """Ignore all dm command errors since commands are not allowed in dms.""" + if isinstance(error, commands.CheckFailure) and isinstance(ctx.channel, discord.DMChannel): + error.handled = True + + +def setup(bot: "ModmailBot") -> None: + """Adds the Tickets cog to the bot.""" + bot.add_cog(TicketsCog(bot)) diff --git a/modmail/utils/threads/__init__.py b/modmail/utils/threads/__init__.py new file mode 100644 index 00000000..ed2a010b --- /dev/null +++ b/modmail/utils/threads/__init__.py @@ -0,0 +1,3 @@ +from modmail.utils.threads.decorators import is_modmail_thread +from modmail.utils.threads.errors import ThreadAlreadyExistsError, ThreadException, ThreadNotFoundError +from modmail.utils.threads.models import MessageDict, Target, Ticket diff --git a/modmail/utils/threads/decorators.py b/modmail/utils/threads/decorators.py new file mode 100644 index 00000000..4d2a5db7 --- /dev/null +++ b/modmail/utils/threads/decorators.py @@ -0,0 +1,24 @@ +from typing import Callable + +from discord.ext import commands +from discord.ext.commands import Context +from discord.threads import Thread + + +def is_modmail_thread() -> Callable: + """Check to see whether the channel in which the command is invoked is a discord thread or not.""" + + def predicate(ctx: Context) -> bool: + """ + Check contextual channel is a modmail thread channel. + + All modmail thread channels are a thread, so if it isn't we know we can stop checking at that point. + If it is a thread channel, then we also know it must have a parent attribute, so we can safely + check if the id is the same as the configured thread log channel id. + """ + return ( + isinstance(ctx.channel, Thread) + and ctx.channel.parent.id == ctx.bot.config.user.threads.relay_channel_id + ) + + return commands.check(predicate) diff --git a/modmail/utils/threads/errors.py b/modmail/utils/threads/errors.py new file mode 100644 index 00000000..42797493 --- /dev/null +++ b/modmail/utils/threads/errors.py @@ -0,0 +1,16 @@ +class ThreadException(Exception): # noqa: N818 + """The base error for threads, all threads errors should inherit from this exception.""" + + pass + + +class ThreadNotFoundError(ThreadException): + """Raised when a thread is not found.""" + + pass + + +class ThreadAlreadyExistsError(ThreadException): + """Raised when a thread already exists.""" + + pass diff --git a/modmail/utils/threads/models.py b/modmail/utils/threads/models.py new file mode 100644 index 00000000..2fb57b37 --- /dev/null +++ b/modmail/utils/threads/models.py @@ -0,0 +1,83 @@ +import logging +from enum import IntEnum, auto +from typing import TYPE_CHECKING, List, Optional, Union + +import discord + + +if TYPE_CHECKING: # pragma: nocover + from modmail.log import ModmailLogger +logger: "ModmailLogger" = logging.getLogger(__name__) + + +class Target(IntEnum): + """Targets for thread messages.""" + + USER = auto() + MODMAIL = auto() + + +class MessageDict(dict): + """ + A dict that stores only discord.Messages as pairs which can be mapped to each other. + + This is implemented by storing the ids as the keys, and the messages as the values. + + Both adding and deleting items will delete both keys, + so the user does not have to worry about managing that. + """ + + def __setitem__(self, key: discord.Message, value: discord.Message): + if not isinstance(key, discord.Message) or not isinstance(value, discord.Message): + raise ValueError("key or value are not of type discord.Message") + super().__setitem__(key.id, value) + super().__setitem__(value.id, key) + + def __getitem__(self, key: Union[discord.Message, int]) -> discord.Message: + return super().__getitem__(getattr(key, "id", key)) + + def __delitem__(self, key: Union[discord.Message, int]) -> None: + super().__delitem__(self.__getitem__(key).id) + return super().__delitem__(getattr(key, "id", key)) + + +class Ticket: + """ + Represents a ticket. + + This class represents a ticket for Modmail. A ticket is a way to send + messages to a specific user. + """ + + recipient: discord.User + thread: discord.Thread + messages: MessageDict + log_message: discord.Message + close_after: Optional[int] = None + last_sent_messages: List[discord.Message] = list() + has_sent_initial_message: bool + + def __init__( + self, + recipient: discord.User, + thread: discord.Thread, + *, + has_sent_initial_message: bool = True, + log_message: discord.Message = None, + ): + """ + Creates a Ticket instance. + + At least thread and user are required. + log_message and close_after are automatically gathered from the thread object + """ + self.thread = thread + self.recipient = recipient + self.log_message: Union[ + discord.Message, discord.PartialMessage + ] = log_message or self.thread.parent.get_partial_message(self.thread.id) + self.messages = MessageDict() + self.close_after = self.thread.auto_archive_duration + self.has_sent_initial_message = has_sent_initial_message + + logger.trace(f"Created a Ticket object for recipient {recipient} with thread {thread}.") diff --git a/modmail/utils/users.py b/modmail/utils/users.py new file mode 100644 index 00000000..18fd2abf --- /dev/null +++ b/modmail/utils/users.py @@ -0,0 +1,18 @@ +import discord + + +async def check_can_dm_user(user: discord.User) -> bool: + """ + Checks if the user has a DM open. + + NOTE: This method has a 100% error rate, so it should not be used unless absolutely necessary. + Repeated usage can lead to a discord api ban. + """ + try: + await user.send("") + except discord.errors.Forbidden: + return False + except discord.errors.HTTPException as e: + return "empty message" in e.text.lower() + except discord.errors.DiscordException as e: + raise e diff --git a/tests/modmail/extensions/test_threads.py b/tests/modmail/extensions/test_threads.py new file mode 100644 index 00000000..d6b45e55 --- /dev/null +++ b/tests/modmail/extensions/test_threads.py @@ -0,0 +1,349 @@ +import typing +import unittest.mock +from typing import TYPE_CHECKING + +import arrow +import discord +import pytest + +from modmail.extensions import threads +from modmail.utils import threads as thread_utils +from tests import mocks + + +if TYPE_CHECKING: # pragma: nocover + from modmail.bot import ModmailBot + +GUILD_ID = mocks.generate_realistic_id() + + +def _get_fake_ticket(): + guild = mocks.MockGuild(id=GUILD_ID) + channel = mocks.MockTextChannel(guild=guild) + user = mocks.MockUser() + thread = mocks.MockThread(guild=guild) + thread.parent = channel + message = mocks.MockMessage(guild=guild, channel=channel) + + ticket = thread_utils.Ticket(user, thread, log_message=message) + return ticket + + +@pytest.fixture() +def ticket(): + """Fixture for ticket generation.""" + return _get_fake_ticket() + + +@pytest.fixture() +def ticket_dict(): + """Create a dictionary with some fake tickets in it.""" + ticket_dict: typing.Dict[int, thread_utils.Ticket] = dict() + for _ in range(2): + tick = _get_fake_ticket() + ticket_dict[tick.recipient.id] = tick + ticket_dict[tick.thread.id] = tick + + return ticket_dict + + +@pytest.fixture() +def bot(): + """ + Fixture for mock bot. + + A few attributes on the bot are set to actual objects instead of remaining as mocks. + """ + bot: ModmailBot = mocks.MockBot() + bot._tickets = dict() + return bot + + +@pytest.fixture() +def cog(bot): + """Fixture of a TicketsCog to make testing easier.""" + cog = threads.TicketsCog(bot) + yield cog + cog.cog_unload() + + +class TestUtilityMethods: + """Test utility methods of the cog that don't fit anywhere else.""" + + @pytest.mark.asyncio + async def test_add_ticket(self, cog: threads.TicketsCog, ticket: threads.Ticket): + """Ensure that add ticket adds the ticket to the dictionary on both sides, like a MessageDict.""" + bot = cog.bot + ticket_copy = await cog.add_ticket(ticket) + + assert ticket_copy is ticket + + assert ticket in bot._tickets.values() + assert ticket.recipient.id in bot._tickets.keys() + assert ticket.thread.id in bot._tickets.keys() + + assert bot._tickets[ticket.thread.id] == ticket + assert bot._tickets[ticket.recipient.id] == ticket + + @pytest.mark.asyncio + async def test_get_ticket(self, bot, cog: threads.TicketsCog, ticket: threads.Ticket): + """Ensure that get_tickets returns the correct ticket.""" + bot._tickets = dict() + await cog.add_ticket(ticket) + received_ticket = await cog.fetch_ticket(ticket.thread.id) + assert ticket is received_ticket + del received_ticket + + received_ticket = await cog.fetch_ticket(ticket.recipient.id) + assert ticket is received_ticket + + @pytest.mark.parametrize("should_raise", [True, False]) + @pytest.mark.asyncio + async def test_invalid_get_ticket(self, cog: threads.TicketsCog, ticket_dict: dict, should_raise: bool): + """Test invalid get_ticket ids raise a ThreadNotFoundError.""" + cog.bot._tickets = ticket_dict + try: + result = await cog.fetch_ticket(mocks.generate_realistic_id(), raise_exception=should_raise) + except threads.ThreadNotFoundError: + assert should_raise + else: + assert not should_raise + assert result is None + + # TODO: More tests for this method + @pytest.mark.asyncio + async def test_create_ticket(self, bot: "ModmailBot", cog: threads.TicketsCog, ticket: threads.Ticket): + """Test the create_ticket method adds the ticket to the internal dictionary.""" + # this method uses a mock ticket since the ticket already has thread, user, log_message. + msg = mocks.MockMessage(guild=None, author=ticket.recipient) + + # patch start discord thread + mock_start_thread_response = (ticket.thread, ticket.log_message) + with unittest.mock.patch.object( + cog, "_start_discord_thread", return_value=mock_start_thread_response + ) as mock_start_discord_thread: + returned_ticket = await cog.create_ticket(msg) + + assert 1 == mock_start_discord_thread.call_count + assert msg == mock_start_discord_thread.call_args[0][0] + assert ticket.recipient == mock_start_discord_thread.call_args[0][1] + + assert ticket.thread is returned_ticket.thread + assert ticket.recipient is returned_ticket.recipient + + assert ticket.thread.id in bot._tickets.keys() + assert ticket.recipient.id in bot._tickets.keys() + assert returned_ticket in bot._tickets.values() + + # TODO: write more tests for this specific method + @pytest.mark.asyncio + async def test_start_discord_thread(self, bot, cog: threads.TicketsCog, ticket: threads.Ticket): + """Test _start_discord_thread does what it says and returns the correct thread and message.""" + cog.relay_channel = mocks.MockTextChannel() + user = mocks.MockUser(name="NitroScammer") + thread = ticket.thread + msg = mocks.MockMessage(guild=None, author=user) + relayed_msg = mocks.MockMessage(guild=thread.guild, channel=cog.relay_channel) + cog.relay_channel.send = unittest.mock.AsyncMock(return_value=relayed_msg) + relayed_msg.create_thread = unittest.mock.AsyncMock(return_value=thread) + + with unittest.mock.patch.object(cog, "init_relay_channel"): + result = await cog._start_discord_thread(msg) + + assert 2 == len(result) + assert thread is result[0] + assert relayed_msg is result[1] + + assert 1 == cog.relay_channel.send.call_count + assert 1 == relayed_msg.create_thread.call_count + + assert str(user) == relayed_msg.create_thread.call_args[1]["name"] + + +@pytest.fixture +def ctx(): + """Mock ctx fixture.""" + return mocks.MockContext() + + +class TestContactCommand: + """Test the contact command which will create a ticket with a specified user.""" + + @pytest.mark.asyncio + async def test_contact(self, ctx, bot, cog, ticket): + """Test a contact command succeeds and creates a thread if everything is accurate.""" + user = mocks.MockUser(name="spammer") + with unittest.mock.patch.object(cog, "create_ticket", return_value=ticket) as mock_create_ticket: + with unittest.mock.patch.object( + threads, "check_can_dm_user", return_value=True + ) as mock_check_can_dm_user: + await cog.contact(cog, ctx, user) + + assert 0 == ticket.thread.send.call_count + assert 1 == mock_create_ticket.call_count + assert 1 == mock_check_can_dm_user.call_count + + mock_create_ticket.assert_called_once_with( + ctx.message, + recipient=user, + raise_for_preexisting=True, + send_initial_message=False, + description="", + creator=ctx.message.author, + ) + + @pytest.mark.asyncio + async def test_does_not_allow_bots(self, ctx, bot, cog): + """Properly notify the user that tickets cannot be started with other bots.""" + user = mocks.MockUser(name="defintely a human", bot=True) + await cog.contact(cog, ctx, user) + assert 1 == ctx.send.call_count + assert "bot" in ctx.send.call_args[0][0] + + ctx.reset_mock() + + await cog.contact(cog, ctx, bot.user) + assert 1 == ctx.send.call_count + assert "perfect" in ctx.send.call_args[0][0] + + @pytest.mark.asyncio + async def test_ticket_already_exists(self, ctx, bot, cog, ticket): + """Send an alert to the user if the thread already exists.""" + user = mocks.MockUser(name="spammer") + with unittest.mock.patch.object( + cog, "create_ticket", side_effect=threads.ThreadAlreadyExistsError() + ) as mock_create_ticket: + with unittest.mock.patch.object(cog, "fetch_ticket", return_value=ticket): + await cog.contact(cog, ctx, user) + thread = ticket.thread + assert 0 == ticket.thread.send.call_count + assert 1 == ctx.send.call_count + assert 1 == mock_create_ticket.call_count + + sent_content = ctx.send.call_args[0][0] + important_info = [ + "already exists", + user.mention, + user.id, + f"{threads.BASE_JUMP_URL}/{thread.guild.id}/{thread.id}", + ] + for item in important_info: + assert str(item) in sent_content + + @pytest.mark.asyncio + async def test_check_can_dm_user_false(self, ctx, bot, cog, ticket): + """Check if the thread is notified if a user is unable to be dmed upon creation.""" + user = mocks.MockUser(name="spammer", discriminator="5555") + user.__str__ = lambda user: user.name + "#" + user.discriminator + + with unittest.mock.patch.object(cog, "create_ticket", return_value=ticket) as mock_create_ticket: + with unittest.mock.patch.object( + threads, "check_can_dm_user", return_value=False + ) as mock_check_can_dm_user: + await cog.contact(cog, ctx, user) + + assert 1 == mock_create_ticket.call_count + assert 1 == mock_check_can_dm_user.call_count + assert 1 == ticket.thread.send.call_count + + mock_create_ticket.assert_called_once_with( + ctx.message, + recipient=user, + raise_for_preexisting=True, + send_initial_message=False, + description="", + creator=ctx.message.author, + ) + # check for important words to be in the reply when user cannot be dmed + sent_text = ticket.thread.send.call_args[0][0] + important_words = ["not able to", "DM", user.name, user.discriminator] + for word in important_words: + assert str(word) in sent_text + + +class TestRelayMessageToUser: + """ + Relay a message from guild to user and save it to the ticket. + + This should: + - send message in thread + - send message to user + - error if the user cannot be dmed (TODO) + - handle all of the following: + - message content + - message stickers + - message attachments + """ + + ... + + @staticmethod + @pytest.fixture + def message(): + """Mock Message.""" + return mocks.MockMessage() + + @pytest.mark.asyncio + @pytest.mark.parametrize(["contents", "should_delete"], [["...", True]]) + async def test_reply_to_user_general( + self, + cog: threads.TicketsCog, + ticket: threads.Ticket, + message: typing.Union[discord.Message, mocks.MockMessage], + contents: str, + should_delete: bool, + ): + """Test the reply to user method does indeed send a message to the user.""" + message.author.colour = 5 + message.created_at = arrow.get(discord.utils.snowflake_time(message.created_at).timestamp()).datetime + + import modmail.utils.embeds + + modmail.utils.embeds.patch_embed() + await cog.relay_message_to_user(ticket, message, contents, delete=should_delete) + + assert 1 == ticket.recipient.send.call_count + ticket.recipient.send.assert_called_once() + ticket.recipient.send.assert_awaited_once() + + +class TestRelayMessageToGuild: + """ + Relay a message from guild to user and save it to the ticket. + + This should: + - send message in thread + - react to the user's message with a confirmation emoji that the above was successful + - handle all of the following: + - message content + - message stickers + - message attachments + """ + + ... + + +class TestReplyCommand: + """ + Test reply command. + + Reply command needs to + - respond to the user with errors + - call the relay message to user method + """ + + ... + + +# class TestEditCommand: +# ... +# class TestDeleteCommand: +# ... +# class TestOnMessage: +# ... +# class TestOnMessageEdit: +# ... +# class TestOnMessageDelete: +# ... +# class TestOnTyping: +# ... diff --git a/tests/modmail/utils/threads/test_decorators.py b/tests/modmail/utils/threads/test_decorators.py new file mode 100644 index 00000000..470301ab --- /dev/null +++ b/tests/modmail/utils/threads/test_decorators.py @@ -0,0 +1,40 @@ +import pytest + +from modmail.utils.threads import decorators +from tests import mocks + + +@pytest.fixture +def is_modmail_thread(): + """Fixture for is_modmail_thread of decorators to return the check it creates.""" + + def func(): + pass + + check = decorators.is_modmail_thread() + check(func) + return func.__commands_checks__[-1] + + +def threaded_ctx(channel_id=None, /): + """Return a ctx object from a thread with a parent channel property on the thread.""" + ctx = mocks.MockContext(channel=mocks.MockThread()) + ctx.channel.parent = mocks.MockTextChannel(id=channel_id or mocks.generate_realistic_id()) + return ctx + + +@pytest.mark.parametrize( + ["ctx", "expected", "config_id"], + [ + [threaded_ctx(42), True, 42], + [threaded_ctx(42), False, 21], + [mocks.MockContext(channel=mocks.MockTextChannel(id=1)), False, 123], + [mocks.MockContext(channel=mocks.MockTextChannel(id=123)), False, 123], + ], +) +def test_is_modmail_thread(ctx, is_modmail_thread, expected: bool, config_id: int): + """Check that is_modmail_thread requires the channel to be a thread and with a parent of log channel.""" + ctx.bot.config.user.threads.relay_channel_id = config_id + result = is_modmail_thread(ctx) + + assert expected == result diff --git a/tests/modmail/utils/threads/test_models.py b/tests/modmail/utils/threads/test_models.py new file mode 100644 index 00000000..eff9c0cd --- /dev/null +++ b/tests/modmail/utils/threads/test_models.py @@ -0,0 +1,106 @@ +import enum +import typing +from functools import cached_property +from tokenize import maybe + +import discord +import pytest + +import modmail +from modmail.utils.threads import models +from tests import mocks + + +class TestMessageDict: + """ + Custom message dictionary needs to store everything as a key and value. + + However, the keys are the ids of the passed values, and support + getting either a corresponding key or message. + """ + + @pytest.fixture(scope="class") + def messages(self): + """ + Return a list of tuples of generated messages. + + The first msg in each tuple has a guild, while the second does not. + The guild is the same mock accross all of the messages. + """ + messages = [] + guild = mocks.MockGuild() + for _ in range(7): + messages.append([mocks.MockMessage(guild=guild), mocks.MockMessage(guild=None)]) + return messages + + def test_setitem(self, messages) -> models.MessageDict: + """__setitem__ should set both the key and the value.""" + msg_dict = models.MessageDict() + + for m1, m2 in messages: + msg_dict[m1] = m2 + + # multiplying by 2 here because messages is a list of tuples + assert len(messages) * 2 == len(msg_dict.keys()) + + for m1, m2 in messages: + assert m1.id in msg_dict.keys() + assert m2.id in msg_dict.keys() + assert m1 in msg_dict.values() + assert m2 in msg_dict.values() + + # for daisy chaining the tests + return msg_dict + + def test_getitem(self, messages): + """__getitem__ should support being able to take either messages or ints as keys.""" + msg_dict = self.test_setitem(messages) + for m1, m2 in messages: + assert m2 == msg_dict[m1] + assert m1 == msg_dict[m2] + + for m1, m2 in messages: + assert m2 == msg_dict[m1.id] + assert m1 == msg_dict[m2.id] + + def test_delitem(self, messages): + """__delitem__ should delete both the key and the matching value.""" + msg_dict = self.test_setitem(messages) + for m1, m2 in messages: + + del msg_dict[m1] + assert m1 not in msg_dict.values() + assert m2.id not in msg_dict.keys() + assert m2 not in msg_dict.values() + + invalid_messages = [ + -1, + "discord.Message", + 806247348703068210, + mocks.MockTextChannel(), + mocks.MockDMChannel(), + ] + + @pytest.mark.parametrize("n1", invalid_messages) + @pytest.mark.parametrize("n2", invalid_messages) + def test_discord_message_required(self, n1, n2): + """Test that MessageDict only takes messages as keys and values in assignment.""" + msg_dict = models.MessageDict() + with pytest.raises(ValueError, match=r"discord\.Message"): + msg_dict[n1] = n2 + + +class TestTicket: + """Tests for models.Ticket.""" + + def test_ticket_attributes(self): + """Tickets should have these attributes as part of their public api with the proper objects.""" + user = mocks.MockUser() + thread = mocks.MockThread() + message = mocks.MockMessage(id=thread.id) + ticket = models.Ticket(user, thread, log_message=message) + + assert user == ticket.recipient + assert thread == ticket.thread + assert message == ticket.log_message + assert isinstance(ticket.messages, models.MessageDict) diff --git a/tox.ini b/tox.ini index 405a6e64..56e5d629 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,7 @@ ignore= per-file-ignores= tests/*:,ANN,S101,F401 docs.py:B008 + modmail/utils/threads/__init__.py:F401 [isort] profile=black From a89f5d672d48560b79d0a90b354e7f84da5d2328 Mon Sep 17 00:00:00 2001 From: arl Date: Wed, 20 Apr 2022 23:27:46 -0400 Subject: [PATCH 99/99] tests/test error handler (#94) * tests: test the first half of the error handler Signed-off-by: onerandomusername * fix: remove duped reference mock changes now mean that ctx.message.channel and ctx.channel are the same object * tests: error handler test reset cooldown * tests: finish testing handle_bot_missing_perms * tests: finish testing handle_user_input_error * tests: add on_command_error tests * fix: move handling command invoke errors to its own function * tests: add tests for command invoke errors * chore: remove unused import * nit: remove unused class with error hierarchy * fix: use type() instead of __class__ --- modmail/extensions/utils/error_handler.py | 80 +-- .../extensions/utils/test_error_handler.py | 470 +++++++++++++++--- 2 files changed, 441 insertions(+), 109 deletions(-) diff --git a/modmail/extensions/utils/error_handler.py b/modmail/extensions/utils/error_handler.py index 181519c6..ae44116f 100644 --- a/modmail/extensions/utils/error_handler.py +++ b/modmail/extensions/utils/error_handler.py @@ -23,6 +23,8 @@ ANY_DEV_MODE = BOT_MODE & (BotModes.DEVELOP.value + BotModes.PLUGIN_DEV.value) +MAYBE_DM_ON_PERM_ERROR = True + class ErrorHandler(ModmailCog, name="Error Handler"): """Handles all errors across the bot.""" @@ -43,7 +45,7 @@ def get_title_from_name(error: typing.Union[Exception, str]) -> str: Eg NSFWChannelRequired returns NSFW Channel Required """ if not isinstance(error, str): - error = error.__class__.__name__ + error = type(error).__name__ return re.sub(ERROR_TITLE_REGEX, r" \1", error) @staticmethod @@ -70,13 +72,12 @@ async def handle_user_input_error( async def handle_bot_missing_perms( self, ctx: commands.Context, error: commands.BotMissingPermissions ) -> bool: - """Handles bot missing permissing by dming the user if they have a permission which may be able to fix this.""" # noqa: E501 + """Handles bot missing permissions by dming the user if they have a permission which may be able to fix this.""" # noqa: E501 embed = self.error_embed("Permissions Failure", str(error)) bot_perms = ctx.channel.permissions_for(ctx.me) - not_responded = True if bot_perms >= discord.Permissions(send_messages=True, embed_links=True): await ctx.send(embeds=[embed]) - not_responded = False + return True elif bot_perms >= discord.Permissions(send_messages=True): # make a message as similar to the embed, using as few permissions as possible # this is the only place we send a standard message instead of an embed @@ -85,7 +86,7 @@ async def handle_bot_missing_perms( "**Permissions Failure**\n\n" "I am missing the permissions required to properly execute your command." ) - # intentionally not setting responded to True, since we want to attempt to dm the user + # intentionally skipping setting responded to True, since we want to attempt to dm the user logger.warning( f"Missing partial required permissions for {ctx.channel}. " "I am able to send messages, but not embeds." @@ -93,13 +94,13 @@ async def handle_bot_missing_perms( else: logger.error(f"Unable to send an error message to channel {ctx.channel}") - if not_responded and ANY_DEV_MODE: + if MAYBE_DM_ON_PERM_ERROR or ANY_DEV_MODE: # non-general permissions perms = discord.Permissions( administrator=True, - manage_threads=True, - manage_roles=True, manage_channels=True, + manage_roles=True, + manage_threads=True, ) if perms.value & ctx.channel.permissions_for(ctx.author).value: logger.info( @@ -111,7 +112,8 @@ async def handle_bot_missing_perms( except discord.Forbidden: logger.notice("Also encountered an error when trying to reply in dms.") return False - return True + else: + return True async def handle_check_failure( self, ctx: commands.Context, error: commands.CheckFailure @@ -135,6 +137,38 @@ async def handle_check_failure( embed = self.error_embed(title, str(error)) return embed + async def handle_command_invoke_error( + self, ctx: commands.Context, error: commands.CommandInvokeError + ) -> typing.Optional[discord.Embed]: + """Formulate an embed for a generic error handler.""" + if isinstance(error.original, discord.Forbidden): + logger.warn(f"Permissions error occurred in {ctx.command}.") + await self.handle_bot_missing_perms(ctx, error.original) + return None + + # todo: this should properly handle plugin errors and note that they are not bot bugs + # todo: this should log somewhere else since this is a bot bug. + # generic error + logger.error(f'Error occurred in command "{ctx.command}".', exc_info=error.original) + if ctx.command.cog.__module__.startswith("modmail.plugins"): + # plugin msg + title = "Plugin Internal Error Occurred" + msg = ( + "Something went wrong internally in the plugin contributed command you were trying " + "to execute. Please report this error and what you were trying to do to the " + "respective plugin developers.\n\n**PLEASE NOTE**: Modmail developers will not help " + "you with this issue and will refer you to the plugin developers." + ) + else: + # built in command msg + title = "Internal Error" + msg = ( + "Something went wrong internally in the command you were trying to execute. " + "Please report this error and what you were trying to do to the bot developers." + ) + logger.debug(ctx.command.callback.__module__) + return self.error_embed(title, msg) + @ModmailCog.listener() async def on_command_error(self, ctx: commands.Context, error: commands.CommandError) -> None: """Activates when a command raises an error.""" @@ -174,33 +208,9 @@ async def on_command_error(self, ctx: commands.Context, error: commands.CommandE embed = self.error_embed("Command Disabled", msg) elif isinstance(error, commands.CommandInvokeError): - if isinstance(error.original, discord.Forbidden): - logger.warn(f"Permissions error occurred in {ctx.command}.") - await self.handle_bot_missing_perms(ctx, error.original) + embed = await self.handle_command_invoke_error(ctx, error) + if embed is None: should_respond = False - else: - # todo: this should properly handle plugin errors and note that they are not bot bugs - # todo: this should log somewhere else since this is a bot bug. - # generic error - logger.error(f'Error occurred in command "{ctx.command}".', exc_info=error.original) - if ctx.command.cog.__module__.startswith("modmail.plugins"): - # plugin msg - title = "Plugin Internal Error Occurred" - msg = ( - "Something went wrong internally in the plugin contributed command you were trying " - "to execute. Please report this error and what you were trying to do to the " - "respective plugin developers.\n\n**PLEASE NOTE**: Modmail developers will not help " - "you with this issue and will refer you to the plugin developers." - ) - else: - # built in command msg - title = "Internal Error" - msg = ( - "Something went wrong internally in the command you were trying to execute. " - "Please report this error and what you were trying to do to the bot developers." - ) - logger.debug(ctx.command.callback.__module__) - embed = self.error_embed(title, msg) # TODO: this has a fundamental problem with any BotMissingPermissions error # if the issue is the bot does not have permissions to send embeds or send messages... diff --git a/tests/modmail/extensions/utils/test_error_handler.py b/tests/modmail/extensions/utils/test_error_handler.py index 8dbfc676..3adf47e4 100644 --- a/tests/modmail/extensions/utils/test_error_handler.py +++ b/tests/modmail/extensions/utils/test_error_handler.py @@ -11,6 +11,10 @@ from tests import mocks +# set dev mode for the cog to a truthy value +error_handler.ANY_DEV_MODE = 2 + + @pytest.fixture def cog(): """Pytest fixture for error_handler.""" @@ -20,7 +24,25 @@ def cog(): @pytest.fixture def ctx(): """Pytest fixture for MockContext.""" - return mocks.MockContext(channel=mocks.MockTextChannel()) + return mocks.MockContext() + + +@pytest.fixture +def command(): + """Fixture for discord.ext.commands.Command.""" + command = unittest.mock.NonCallableMock(spec_set=commands.Command(unittest.mock.AsyncMock(), name="mock")) + return command + + +@pytest.mark.parametrize("is_cooldown", [True, False]) +def test_reset_cooldown(ctx, cog, is_cooldown: bool): + """Test the cooldown is reset if the command is on a cooldown.""" + ctx.command.is_on_cooldown.return_value = bool(is_cooldown) + cog._reset_command_cooldown(ctx) + assert 1 == ctx.command.is_on_cooldown.call_count + assert int(is_cooldown) == ctx.command.reset_cooldown.call_count + if int(is_cooldown) == 1: + ctx.command.reset_cooldown.assert_called_once_with(ctx) def test_error_embed(): @@ -48,6 +70,7 @@ def test_get_title_from_name(exception_or_str: typing.Union[Exception, str], exp assert expected_str == result +@pytest.mark.parametrize("reset_cooldown", [True, False]) @pytest.mark.parametrize( ["error", "title", "description"], [ @@ -66,109 +89,408 @@ def test_get_title_from_name(exception_or_str: typing.Union[Exception, str], exp "Guild Not Found", 'Guild "ImportantGuild" not found.', ], + [ + commands.BadUnionArgument( + inspect.Parameter("colour", 2), + (commands.InviteConverter, commands.ColourConverter), + [commands.BadBoolArgument("colour"), commands.BadColourArgument("colour")], + ), + "Bad Union Argument", + 'Could not convert "colour" into Invite Converter or Colour Converter.', + ], ], ) @pytest.mark.asyncio async def test_handle_user_input_error( - cog: ErrorHandler, ctx: mocks.MockContext, error: commands.UserInputError, title: str, description: str + cog: ErrorHandler, + ctx: mocks.MockContext, + error: commands.UserInputError, + title: str, + description: str, + reset_cooldown: bool, ): - """Test user input errors are handled properly. Does not test with BadUnionArgument.""" - embed = await cog.handle_user_input_error(ctx=ctx, error=error, reset_cooldown=False) + """Test user input errors are handled properly.""" + with unittest.mock.patch.object(cog, "_reset_command_cooldown") as mock_cooldown_reset: + embed = await cog.handle_user_input_error(ctx=ctx, error=error, reset_cooldown=reset_cooldown) assert title == embed.title assert description == embed.description + if reset_cooldown: + assert 1 == mock_cooldown_reset.call_count + +@pytest.mark.parametrize( + ["error", "bot_perms", "should_send_channel", "member_perms", "should_send_user", "raise_forbidden"], + [ + ( + commands.BotMissingPermissions(["manage_guild"]), + discord.Permissions(read_messages=True, send_messages=True, embed_links=True), + 1, + discord.Permissions(read_messages=True, send_messages=True, embed_links=True), + 0, + False, + ), + ( + commands.BotMissingPermissions(["administrator"]), + discord.Permissions(read_messages=True, send_messages=True, manage_guild=True), + 1, + discord.Permissions(read_messages=True, send_messages=True, embed_links=True), + 0, + False, + ), + ( + commands.BotMissingPermissions(["mention_everyone"]), + discord.Permissions(read_messages=True, send_messages=True), + 1, + discord.Permissions(read_messages=True, send_messages=True, embed_links=True), + 0, + False, + ), + ( + commands.BotMissingPermissions(["administrator"]), + discord.Permissions(read_messages=False, send_messages=False), + 0, + discord.Permissions(read_messages=False), + 0, + False, + ), + ( + commands.BotMissingPermissions(["change_nickname"]), + discord.Permissions(read_messages=True, send_messages=True), + 1, + discord.Permissions(read_messages=True, send_messages=True, administrator=True), + 1, + False, + ), + ( + commands.BotMissingPermissions(["administrator"]), + discord.Permissions(manage_threads=True, manage_channels=True), + 0, + discord.Permissions(administrator=True), + 1, + False, + ), + ( + commands.BotMissingPermissions(["change_nickname"]), + discord.Permissions(read_messages=True, send_messages=True), + 1, + discord.Permissions(read_messages=True, send_messages=True, administrator=True), + 1, + True, + ), + ( + commands.BotMissingPermissions(["administrator"]), + discord.Permissions(manage_threads=True, manage_channels=True), + 0, + discord.Permissions(administrator=True), + 1, + True, + ), + ], +) @pytest.mark.asyncio -async def test_handle_bot_missing_perms(cog: ErrorHandler): +async def test_handle_bot_missing_perms( + cog: ErrorHandler, + ctx: mocks.MockContext, + error: commands.BotMissingPermissions, + bot_perms: discord.Permissions, + should_send_channel: int, + member_perms: discord.Permissions, + should_send_user: int, + raise_forbidden: bool, +): """ - Test error_handler.handle_bot_missing_perms. There are some cases here where the bot is unable to send messages, and that should be clear. """ - ... + def mock_permissions_for(member): + assert isinstance(member, discord.Member) + if member is ctx.me: + return bot_perms + if member is ctx.author: + return member_perms + # fail since there is no other kind of user who should be passed here + pytest.fail("An invalid member or role was passed to ctx.channel.permissions_for") + + if raise_forbidden: + error_to_raise = discord.Forbidden(unittest.mock.MagicMock(status=403), "no.") + ctx.author.send.side_effect = error_to_raise + ctx.message.author.send.side_effect = error_to_raise + with unittest.mock.patch.object(ctx.channel, "permissions_for", mock_permissions_for): + + await cog.handle_bot_missing_perms(ctx, error) + + assert should_send_channel == ctx.send.call_count + ctx.channel.send.call_count + + # note: this may break depending on dev-mode and relay mode. + assert should_send_user == ctx.author.send.call_count + + +@pytest.mark.parametrize( + ["error", "expected_title"], + [ + [ + commands.CheckAnyFailure( + ["Something went wrong"], + [commands.NoPrivateMessage(), commands.PrivateMessageOnly()], + ), + "Something went wrong", + ], + [commands.NoPrivateMessage(), "Server Only"], + [commands.PrivateMessageOnly(), "DMs Only"], + [commands.NotOwner(), "Not Owner"], + [commands.MissingPermissions(["send_message"]), "Missing Permissions"], + [commands.BotMissingPermissions(["send_message"]), None], + [commands.MissingRole(mocks.MockRole().id), "Missing Role"], + [commands.BotMissingRole(mocks.MockRole().id), "Bot Missing Role"], + [commands.MissingAnyRole([mocks.MockRole().id]), "Missing Any Role"], + [commands.BotMissingAnyRole([mocks.MockRole().id]), "Bot Missing Any Role"], + [commands.NSFWChannelRequired(mocks.MockTextChannel()), "NSFW Channel Required"], + ], +) @pytest.mark.asyncio -async def test_handle_check_failure(cog: ErrorHandler): +async def test_handle_check_failure( + cog: ErrorHandler, ctx: mocks.MockContext, error: commands.CheckFailure, expected_title: str +): """ Test check failures. In some cases, this method should result in calling a bot_missing_perms method because the bot cannot send messages. """ - ... + with unittest.mock.patch.object(cog, "handle_bot_missing_perms"): + if isinstance(error, commands.BotMissingPermissions): + assert await cog.handle_check_failure(ctx, error) is None + return + embed = await cog.handle_check_failure(ctx, error) + assert embed.title == expected_title -@pytest.mark.asyncio -async def test_on_command_error(cog: ErrorHandler): - """Test the general command error method.""" - ... +class TestCommandInvokeError: + """ + Collection of tests for ErrorHandler.handle_command_invoke_error. -class TestErrorHandler: + This serves as nothing more but to group the tests for the single method """ - Test class for the error handler. The problem here is a lot of the errors need to be raised. - Thankfully, most of them do not have extra attributes that we use, and can be easily faked. + @pytest.mark.asyncio + async def test_forbidden(self, cog: ErrorHandler, ctx: mocks.MockContext): + """Test discord.Forbidden errors are not met with an attempt to send a message.""" + error = commands.CommandInvokeError(unittest.mock.Mock(spec_set=discord.Forbidden)) + with unittest.mock.patch.object(cog, "handle_bot_missing_perms"): + result = await cog.handle_command_invoke_error(ctx, error) + + assert result is None + assert 0 == ctx.send.call_count + + @pytest.mark.parametrize( + ["module", "title_words", "message_words", "exclude_words"], + [ + [ + "modmail.extensions.utils.error_handler", + ["internal", "error"], + ["internally", "wrong", "report", "developers"], + ["plugin"], + ], + [ + "modmail.plugins.better_error_handler.main", + ["plugin", "error"], + ["plugin", "wrong", "plugin developers"], + None, + ], + ], + ) + @pytest.mark.asyncio + async def test_error( + self, + cog: ErrorHandler, + ctx: mocks.MockContext, + command: commands.Command, + module: str, + title_words: list, + message_words: list, + exclude_words: list, + ): + """Test that the proper alerts are shared in the returned embed.""" + embed = discord.Embed(description="you failed") + error = commands.CommandInvokeError(Exception("lul")) + ctx.command = command + + # needs a mock cog for __module__ + mock_cog = unittest.mock.NonCallableMock(spec_set=commands.Cog) + mock_cog.__module__ = module + ctx.command.cog = mock_cog + + def error_embed(title, msg): + """Replace cog.error_embed and test that the correct params are passed.""" + title = title.lower() + for word in title_words: + assert word in title + + msg = msg.lower() + for word in message_words: + assert word in msg + + if exclude_words: + for word in exclude_words: + assert word not in title + assert word not in msg + + return embed + + with unittest.mock.patch.object(cog, "error_embed", side_effect=error_embed): + result = await cog.handle_command_invoke_error(ctx, error) + + assert result is embed + + +class TestOnCommandError: + """ + Collection of tests for ErrorHandler.on_command_error. + + This serves as nothing more but to group the tests for the single method """ - errors = { - commands.CommandError: [ - commands.ConversionError, - { - commands.UserInputError: [ - commands.MissingRequiredArgument, - commands.TooManyArguments, - { - commands.BadArgument: [ - commands.MessageNotFound, - commands.MemberNotFound, - commands.GuildNotFound, - commands.UserNotFound, - commands.ChannelNotFound, - commands.ChannelNotReadable, - commands.BadColourArgument, - commands.RoleNotFound, - commands.BadInviteArgument, - commands.EmojiNotFound, - commands.GuildStickerNotFound, - commands.PartialEmojiConversionFailure, - commands.BadBoolArgument, - commands.ThreadNotFound, - ] - }, - commands.BadUnionArgument, - commands.BadLiteralArgument, - { - commands.ArgumentParsingError: [ - commands.UnexpectedQuoteError, - commands.InvalidEndOfQuotedStringError, - commands.ExpectedClosingQuoteError, - ] - }, - ] - }, - commands.CommandNotFound, - { - commands.CheckFailure: [ - commands.CheckAnyFailure, - commands.PrivateMessageOnly, - commands.NoPrivateMessage, - commands.NotOwner, - commands.MissingPermissions, - commands.BotMissingPermissions, - commands.MissingRole, - commands.BotMissingRole, - commands.MissingAnyRole, - commands.BotMissingAnyRole, - commands.NSFWChannelRequired, - ] - }, - commands.DisabledCommand, - commands.CommandInvokeError, - commands.CommandOnCooldown, - commands.MaxConcurrencyReached, - ] - } + @pytest.mark.asyncio + async def test_ignore_already_handled(self, cog: ErrorHandler, ctx: mocks.MockContext): + """Assert errors handled elsewhere are ignored.""" + error = commands.NotOwner() + error.handled = True + await cog.on_command_error(ctx, error) + + @pytest.mark.asyncio + async def test_ignore_command_not_found(self, cog: ErrorHandler, ctx: mocks.MockContext): + """Test the command handler ignores command not found errors.""" + await cog.on_command_error(ctx, commands.CommandNotFound()) + + assert 0 == ctx.send.call_count + + @pytest.mark.parametrize( + ["error", "delegate", "embed"], + [ + [ + commands.UserInputError("User input the wrong thing I guess, not sure."), + "handle_user_input_error", + discord.Embed(description="si"), + ], + [ + commands.CheckFailure("Checks failed, crosses passed."), + "handle_check_failure", + discord.Embed(description="also si"), + ], + [ + commands.CheckFailure("Checks failed, crosses passed."), + "handle_check_failure", + None, + ], + [ + unittest.mock.NonCallableMock(spec_set=commands.CommandInvokeError), + "handle_command_invoke_error", + discord.Embed(description=""), + ], + [ + unittest.mock.NonCallableMock(spec_set=commands.CommandInvokeError), + "handle_command_invoke_error", + None, + ], + ], + ) + @pytest.mark.asyncio + async def test_errors_delegated( + self, + cog: ErrorHandler, + ctx: mocks.MockContext, + error: commands.CommandError, + delegate: str, + embed: typing.Optional[discord.Embed], + ): + """Test that the main error method delegates errors as appropriate to helper methods.""" + with unittest.mock.patch.object(cog, delegate) as mock: + mock.return_value = embed + await cog.on_command_error(ctx, error) + + assert 1 == mock.call_count + assert unittest.mock.call(ctx, error) == mock.call_args + + assert int(bool(embed)) == ctx.send.call_count + + if embed is None: + return + + assert unittest.mock.call(embeds=[embed]) == ctx.send.call_args + + @pytest.mark.parametrize( + ["embed", "error", "hidden", "disabled_reason"], + [ + [ + discord.Embed(description="hey its me your worst error"), + commands.DisabledCommand("disabled command, yert"), + True, + None, + ], + [ + discord.Embed(description="hey its me your worst error"), + commands.DisabledCommand("disabled command, yert"), + False, + None, + ], + [ + discord.Embed(description="hey its me your worst error"), + commands.DisabledCommand("disabled command, yert"), + False, + "Some message that should show up once the mock is right", + ], + ], + ) + @pytest.mark.asyncio + async def test_disabled_command( + self, + cog: ErrorHandler, + ctx: mocks.MockContext, + command: commands.Command, + embed: discord.Embed, + error: commands.DisabledCommand, + hidden: bool, + disabled_reason: str, + ): + """Test disabled commands have the right error message.""" + + def error_embed(title: str, message: str): + if disabled_reason: + assert disabled_reason in message + return embed + + ctx.command = command + ctx.invoked_with = command.name + ctx.command.hidden = hidden + ctx.command.extras = dict() + should_respond = not hidden + if disabled_reason: + ctx.command.extras["disabled_reason"] = disabled_reason + + mock = unittest.mock.Mock(side_effect=error_embed) + + with unittest.mock.patch.object(cog, "error_embed", mock): + await cog.on_command_error(ctx, error) + + assert int(should_respond) == ctx.send.call_count + if should_respond: + assert unittest.mock.call(embeds=[embed]) == ctx.send.call_args + + @pytest.mark.asyncio + async def test_default_embed(self, cog, ctx): + """Test the default embed calls the right methods the correct number of times.""" + embed = discord.Embed(description="I need all of the errors!") + error = unittest.mock.NonCallableMock(spec_set=commands.ConversionError) + + with unittest.mock.patch.object(cog, "error_embed") as mock: + mock.return_value = embed + await cog.on_command_error(ctx, error) + + assert 1 == ctx.send.call_count + assert unittest.mock.call(embeds=[embed]) == ctx.send.call_args