From b5c541790c038b2c327af204739b5493774d30b3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 00:40:16 -0400 Subject: [PATCH 01/76] feat: rewrite the entire configuration system to use marshmallow and attrs --- app.json | 2 +- modmail/__init__.py | 6 +- modmail/__main__.py | 2 +- modmail/bot.py | 6 +- modmail/config.py | 248 ++++++++++++++++++++-------------- modmail/utils/embeds.py | 4 +- modmail/utils/extensions.py | 4 +- poetry.lock | 257 +++++++++++++++++++++++------------- pyproject.toml | 10 +- requirements.txt | 19 ++- 10 files changed, 350 insertions(+), 208 deletions(-) diff --git a/app.json b/app.json index 11af2756..2a69903a 100644 --- a/app.json +++ b/app.json @@ -1,6 +1,6 @@ { "env": { - "TOKEN": { + "MODMAIL_BOT_TOKEN": { "description": "Discord bot token. This is from https://discord.com/developers/applications", "required": true } diff --git a/modmail/__init__.py b/modmail/__init__.py index 9377ffd2..8f61401e 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -3,10 +3,14 @@ from pathlib import Path import coloredlogs +import environs from modmail.log import ModmailLogger +env = environs.Env() +env.read_env(".env") + logging.TRACE = 5 logging.NOTICE = 25 logging.addLevelName(logging.TRACE, "TRACE") @@ -17,7 +21,7 @@ # this logging level is set to logging.TRACE because if it is not set to the lowest level, # the child level will be limited to the lowest level this is set to. -ROOT_LOG_LEVEL = logging.TRACE +ROOT_LOG_LEVEL = env.log_level("MODMAIL_LOG_LEVEL", logging.TRACE) FMT = "%(asctime)s %(levelname)10s %(name)15s - [%(lineno)5d]: %(message)s" DATEFMT = "%Y/%m/%d %H:%M:%S" diff --git a/modmail/__main__.py b/modmail/__main__.py index fd10af2a..6c1945f5 100644 --- a/modmail/__main__.py +++ b/modmail/__main__.py @@ -21,7 +21,7 @@ def main() -> None: """Run the bot.""" patch_embed() bot = ModmailBot() - bot.run(bot.config.bot.token) + bot.run(bot.config.user.bot.token) if __name__ == "__main__": diff --git a/modmail/bot.py b/modmail/bot.py index e62f1bda..8d30426f 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -11,7 +11,7 @@ from discord.client import _cleanup_loop from discord.ext import commands -from modmail.config import CONFIG +from modmail.config import config from modmail.log import ModmailLogger from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions from modmail.utils.plugins import PLUGINS, walk_plugins @@ -37,7 +37,7 @@ class ModmailBot(commands.Bot): logger: ModmailLogger = logging.getLogger(__name__) def __init__(self, **kwargs): - self.config = CONFIG + self.config = config self.start_time: t.Optional[arrow.Arrow] = None # arrow.utcnow() self.http_session: t.Optional[ClientSession] = None @@ -45,7 +45,7 @@ def __init__(self, **kwargs): activity = Activity(type=discord.ActivityType.listening, name="users dming me!") # listen to messages mentioning the bot or matching the prefix # ! NOTE: This needs to use the configuration system to get the prefix from the db once it exists. - prefix = commands.when_mentioned_or(CONFIG.bot.prefix) + prefix = commands.when_mentioned_or(self.config.user.bot.prefix) # allow only user mentions by default. # ! NOTE: This may change in the future to allow roles as well allowed_mentions = AllowedMentions(everyone=False, users=True, roles=False, replied_user=True) diff --git a/modmail/config.py b/modmail/config.py index dce1efeb..9dbd1396 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,144 +1,194 @@ -import asyncio -import datetime -import json +import inspect import logging import os +import pathlib import sys import typing -from pathlib import Path -from typing import Any, Dict, Optional, Tuple +from collections import defaultdict +from pprint import pprint +import atoml +import attr +import desert import discord -import toml -from discord.ext.commands import BadArgument -from pydantic import BaseModel -from pydantic import BaseSettings as PydanticBaseSettings -from pydantic import Field, SecretStr -from pydantic.color import Color as ColorBase -from pydantic.env_settings import SettingsSourceCallable -from pydantic.types import conint +import environs +import marshmallow +import marshmallow.fields +import marshmallow.validate -log = logging.getLogger(__name__) +ENV_PREFIX = "MODMAIL_" -CONFIG_PATHS: list = [ - f"{os.getcwd()}/config.toml", - f"{os.getcwd()}/modmail/config.toml", - "./config.toml", -] +env = environs.Env(eager=False, expand_vars=True) -DEFAULT_CONFIG_PATHS = [os.path.join(os.path.dirname(__file__), "config-default.toml")] +DEFAULT_CONFIG_PATH = pathlib.Path(os.path.join(os.path.dirname(__file__), "test.toml")) -def determine_file_path( - paths=typing.Union[list, tuple], config_type: str = "default" -) -> typing.Union[str, None]: - path = None - for file_path in paths: - config_file = Path(file_path) - if (config_file).exists(): - path = config_file - log.debug(f"Found {config_type} config at {file_path}") - break - return path or None +def _generate_default_dict(): + """For defaultdicts to default to a defaultdict.""" + return defaultdict(_generate_default_dict) -DEFAULT_CONFIG_PATH = determine_file_path(DEFAULT_CONFIG_PATHS) -USER_CONFIG_PATH = determine_file_path(CONFIG_PATHS, config_type="") +try: + with open(DEFAULT_CONFIG_PATH) as f: + unparsed_user_provided_cfg = defaultdict(_generate_default_dict, atoml.parse(f.read()).value) +except FileNotFoundError: + unparsed_user_provided_cfg = defaultdict(_generate_default_dict) -def toml_default_config_source(settings: PydanticBaseSettings) -> Dict[str, Any]: - """ - A simple settings source that loads variables from a toml file - from within the module's source folder. - Here we happen to choose to use the `env_file_encoding` from Config - when reading `config-default.toml` - """ - return dict(**toml.load(DEFAULT_CONFIG_PATH)) +class _ColourField(marshmallow.fields.Field): + """Class to convert a str or int into a color and deseriaze into a string.""" + + def _serialize(self, value: typing.Any, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: + return str(value.value) + + def _deserialize( + self, + value: typing.Any, + attr: typing.Optional[str], + data: typing.Optional[typing.Mapping[str, typing.Any]], + **kwargs, + ) -> str: + if not isinstance(value, discord.Colour): + value = discord.Colour(int(value, 16)) + return value + + +# marshmallow.fields.Colour = _ColourField -def toml_user_config_source(settings: PydanticBaseSettings) -> Dict[str, Any]: +@attr.s(auto_attribs=True, frozen=True) +class Bot: + """Values that are configuration for the bot itself. + + These are metavalues, and are the token, prefix, database bind, basically all of the stuff that needs to + be known BEFORE attempting to log in to the database or discord. """ - A simple settings source that loads variables from a toml file - from within the module's source folder. - Here we happen to choose to use the `env_file_encoding` from Config - when reading `config-default.toml` + token: str = attr.ib( + default=marshmallow.missing, + metadata={ + "required": True, + "load_only": True, + "allow_none": False, + }, + ) + prefix: str = attr.ib( + default="?", + metadata={ + "allow_none": False, + }, + ) + + class Meta: + load_only = ("token",) + partial = True + + +def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: + if isinstance(col, discord.Colour): + return col + if isinstance(col, str): + col = int(col, 16) + return discord.Colour(col) + + +@attr.s(auto_attribs=True) +class BotModeCfg: + production: bool = desert.ib( + marshmallow.fields.Constant(True), default=True, metadata={"dump_default": True, "dump_only": True} + ) + develop: bool = attr.ib(default=False, metadata={"allow_none": False}) + plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) + + +@attr.s(auto_attribs=True) +class Colours: """ - if USER_CONFIG_PATH: - return dict(**toml.load(USER_CONFIG_PATH)) - else: - return dict() + Default colors. + These should only be changed here to change the default colors. + """ -class BaseSettings(PydanticBaseSettings): - class Config: - extra = "ignore" - env_file = ".env" - env_file_encoding = "utf-8" + base_embed_color: discord.Colour = desert.ib( + _ColourField(), default="0x7289DA", converter=convert_to_color + ) - @classmethod - def customise_sources( - cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - return ( - env_settings, - init_settings, - file_secret_settings, - toml_user_config_source, - toml_default_config_source, - ) +@attr.s(auto_attribs=True) +class DevCfg: + mode: BotModeCfg = BotModeCfg() + log_level: int = desert.ib( + marshmallow.fields.Integer( + validate=marshmallow.validate.Range(0, 50, error="Logging level must be within 0 to 50.") + ), + default=logging.INFO, + ) -class BotConfig(BaseSettings): - prefix: str = "?" - token: str = None - class Config: - # env_prefix = "bot." - allow_mutation = False +@attr.s(auto_attribs=True) +class Cfg: + bot: Bot = Bot() + colours: Colours = Colours() + dev: DevCfg = DevCfg() + class Meta: + exclude = ("bot.token",) -class BotMode(BaseSettings): + +# find and build a bot class from our env +def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing.Dict = None) -> Bot: """ - Bot mode. + Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. - Used to determine when the bot will run. + If defaults is provided, uses a value from there if the environment variable is not set or is None. """ + # get the attributes of the provided class + if defaults is None: + defaults = defaultdict(lambda: marshmallow.missing) + else: + defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) + attribs: typing.Set[attr.Attribute] = set() + for a in dir(klass.__attrs_attrs__): + if hasattr(klass.__attrs_attrs__, a): + if isinstance(attribute := getattr(klass.__attrs_attrs__, a), attr.Attribute): + attribs.add(attribute) + # read the env vars from the above + with env.prefixed(ENV_PREFIX): + kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing + for var in attribs: + kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) + if defaults and kw[var.name] is None: + kw[var.name] = defaults[var.name] - production: bool = True - plugin_dev: bool = False - develop: bool = False + return klass(**kw) -class Colors(BaseSettings): - """ - Default colors. +unparsed_user_provided_cfg["bot"] = attr.asdict( + _build_bot_class(Bot, "BOT_", unparsed_user_provided_cfg["bot"]) +) - These should only be changed here to change the default colors. - """ +# env.seal() - embed_color: ColorBase = "0087BD" +# build configuration +Configuration = desert.schema_class(Cfg) # noqa: N818 +USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.RAISE) -class DevConfig(BaseSettings): - """ - Developer specific configuration. - These settings should not be changed unless you know what you're doing. - """ +# hide the bot token from serilzation +# this prevents the token from being saved to places. + + +DEFAULTS_CONFIG = Cfg() - log_level: conint(ge=0, le=50) = getattr(logging, "NOTICE", 25) - mode: BotMode +@attr.s(auto_attribs=True, slots=True) +class Config: + user: Cfg + default: Cfg -class ModmailConfig(BaseSettings): - bot: BotConfig - dev: DevConfig - colors: Colors +config = Config(USER_PROVIDED_CONFIG, DEFAULTS_CONFIG) -CONFIG = ModmailConfig() +print(config.user.bot.prefix) diff --git a/modmail/utils/embeds.py b/modmail/utils/embeds.py index 6e0bb150..1e945c4c 100644 --- a/modmail/utils/embeds.py +++ b/modmail/utils/embeds.py @@ -3,10 +3,10 @@ import discord from discord.embeds import EmptyEmbed -from modmail.config import CONFIG +from modmail.config import config -DEFAULT_COLOR = int(CONFIG.colors.embed_color.as_hex().lstrip("#"), 16) +DEFAULT_COLOR = config.user.colours.base_embed_color original_init = discord.Embed.__init__ diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index be547c50..23f10880 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -8,7 +8,7 @@ import typing as t from modmail import extensions -from modmail.config import CONFIG +from modmail.config import config from modmail.log import ModmailLogger from modmail.utils.cogs import BOT_MODES, BotModes, ExtMetadata @@ -35,7 +35,7 @@ def determine_bot_mode() -> int: """ bot_mode = 0 for mode in BotModes: - if getattr(CONFIG.dev.mode, unqualify(str(mode)).lower(), True): + if getattr(config.user.dev.mode, unqualify(str(mode)).lower(), True): bot_mode += mode.value return bot_mode diff --git a/poetry.lock b/poetry.lock index d0fdf060..86aa99a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -58,6 +58,14 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "atoml" +version = "1.0.3" +description = "Yet another style preserving TOML library" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "attrs" version = "21.2.0" @@ -101,7 +109,7 @@ stevedore = ">=1.20.0" [[package]] name = "black" -version = "21.8b0" +version = "21.9b0" description = "The uncompromising code formatter." category = "dev" optional = false @@ -182,7 +190,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "charset-normalizer" -version = "2.0.4" +version = "2.0.6" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "dev" optional = false @@ -250,6 +258,23 @@ toml = {version = "*", optional = true, markers = "extra == \"toml\""} [package.extras] toml = ["toml"] +[[package]] +name = "desert" +version = "2020.11.18" +description = "Deserialize to objects while staying DRY" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +attrs = "*" +marshmallow = ">=3.0" +typing-inspect = "*" + +[package.extras] +dev = ["coverage", "cuvner", "marshmallow-enum", "marshmallow-union", "pytest", "pytest-cov", "pytest-sphinx", "pytest-travis-fold", "tox", "importlib-metadata", "versioneer", "black", "pylint", "pex", "bump2version", "docutils", "check-manifest", "readme-renderer", "pygments", "isort", "mypy", "towncrier", "twine", "wheel"] +test = ["coverage", "cuvner", "marshmallow-enum", "marshmallow-union", "pytest", "pytest-cov", "pytest-sphinx", "pytest-travis-fold", "tox", "importlib-metadata"] + [[package]] name = "discord.py" version = "2.0.0a0" @@ -269,6 +294,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] [package.source] type = "url" url = "https://github.com/Rapptz/discord.py/archive/master.zip" + [[package]] name = "distlib" version = "0.3.2" @@ -277,6 +303,24 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "environs" +version = "9.3.3" +description = "simplified environment variable parsing" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +marshmallow = ">=3.0.0" +python-dotenv = "*" + +[package.extras] +dev = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url", "flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)", "tox"] +django = ["dj-database-url", "dj-email-url", "django-cache-url"] +lint = ["flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)"] +tests = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url"] + [[package]] name = "execnet" version = "1.9.0" @@ -448,11 +492,11 @@ smmap = ">=3.0.1,<5" [[package]] name = "gitpython" -version = "3.1.20" -description = "Python Git Library" +version = "3.1.24" +description = "GitPython is a python library used to interact with Git repositories" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" @@ -460,18 +504,18 @@ typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\" [[package]] name = "humanfriendly" -version = "9.2" +version = "10.0" description = "Human friendly output for text interfaces using Python" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -pyreadline = {version = "*", markers = "sys_platform == \"win32\""} +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} [[package]] name = "identify" -version = "2.2.13" +version = "2.2.14" description = "File identification library for Python" category = "dev" optional = false @@ -559,6 +603,31 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "marshmallow" +version = "3.13.0" +description = "A lightweight library for converting complex datatypes to and from native Python datatypes." +category = "main" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["pytest", "pytz", "simplejson", "mypy (==0.910)", "flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "pre-commit (>=2.4,<3.0)", "tox"] +docs = ["sphinx (==4.1.1)", "sphinx-issues (==1.2.0)", "alabaster (==0.7.12)", "sphinx-version-warning (==1.1.2)", "autodocsumm (==0.2.6)"] +lint = ["mypy (==0.910)", "flake8 (==3.9.2)", "flake8-bugbear (==21.4.3)", "pre-commit (>=2.4,<3.0)"] +tests = ["pytest", "pytz", "simplejson"] + +[[package]] +name = "marshmallow-enum" +version = "1.5.1" +description = "Enum field for Marshmallow" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +marshmallow = ">=2.0.0" + [[package]] name = "mccabe" version = "0.6.1" @@ -627,14 +696,11 @@ pymdown-extensions = ">=7.0" [[package]] name = "mkdocs-material-extensions" -version = "1.0.1" +version = "1.0.3" description = "Extension pack for Python Markdown." category = "dev" optional = false -python-versions = ">=3.5" - -[package.dependencies] -mkdocs-material = ">=5.0.0" +python-versions = ">=3.6" [[package]] name = "mslex" @@ -656,7 +722,7 @@ python-versions = ">=3.6" name = "mypy-extensions" version = "0.4.3" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -733,7 +799,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "2.14.1" +version = "2.15.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false @@ -796,22 +862,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "pydantic" -version = "1.8.2" -description = "Data validation and settings management using python 3.6 type hinting" -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -python-dotenv = {version = ">=0.10.4", optional = true, markers = "extra == \"dotenv\""} -typing-extensions = ">=3.7.4.3" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - [[package]] name = "pydocstyle" version = "6.1.1" @@ -862,9 +912,9 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] -name = "pyreadline" -version = "2.1" -description = "A python implmementation of GNU readline." +name = "pyreadline3" +version = "3.3" +description = "A python implementation of GNU readline." category = "main" optional = false python-versions = "*" @@ -1078,7 +1128,7 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" [[package]] name = "taskipy" -version = "1.8.1" +version = "1.8.2" description = "tasks runner for python projects" category = "dev" optional = false @@ -1086,7 +1136,7 @@ python-versions = ">=3.6,<4.0" [package.dependencies] colorama = ">=0.4.4,<0.5.0" -mslex = ">=0.3.0,<0.4.0" +mslex = {version = ">=0.3.0,<0.4.0", markers = "sys_platform == \"win32\""} psutil = ">=5.7.2,<6.0.0" toml = ">=0.10.0,<0.11.0" @@ -1115,7 +1165,7 @@ test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybi name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" +category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" @@ -1135,6 +1185,18 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "typing-inspect" +version = "0.7.1" +description = "Runtime inspection utilities for typing module." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +mypy-extensions = ">=0.3.0" +typing-extensions = ">=3.7.4" + [[package]] name = "urllib3" version = "1.26.6" @@ -1150,7 +1212,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.7.2" +version = "20.8.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -1205,7 +1267,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "13d426f2143ed7a3fb374ff734603e7dd6e16bf259834232f9fb6c056f11f2bb" +content-hash = "6cfec48a5437401e5e5aeece5efc90f48eae9e2903d6117c4f2cc850af33cdab" [metadata.files] aiodns = [ @@ -1263,6 +1325,10 @@ atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] +atoml = [ + {file = "atoml-1.0.3-py3-none-any.whl", hash = "sha256:944c0e9043ca4e0729d4125132841ef1110677b8d015a624892d63cdc4988655"}, + {file = "atoml-1.0.3.tar.gz", hash = "sha256:5dd70efcafde94a6aa5db2e8c6af5d832bf95b38f47d3283ee3779e920218e94"}, +] attrs = [ {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, @@ -1276,8 +1342,8 @@ bandit = [ {file = "bandit-1.7.0.tar.gz", hash = "sha256:8a4c7415254d75df8ff3c3b15cfe9042ecee628a1e40b44c15a98890fbfc2608"}, ] black = [ - {file = "black-21.8b0-py3-none-any.whl", hash = "sha256:2a0f9a8c2b2a60dbcf1ccb058842fb22bdbbcb2f32c6cc02d9578f90b92ce8b7"}, - {file = "black-21.8b0.tar.gz", hash = "sha256:570608d28aa3af1792b98c4a337dbac6367877b47b12b88ab42095cfc1a627c2"}, + {file = "black-21.9b0-py3-none-any.whl", hash = "sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115"}, + {file = "black-21.9b0.tar.gz", hash = "sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91"}, ] brotlipy = [ {file = "brotlipy-0.7.0-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2"}, @@ -1361,11 +1427,6 @@ cffi = [ {file = "cffi-1.14.6-cp27-cp27m-win_amd64.whl", hash = "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33"}, - {file = "cffi-1.14.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5"}, - {file = "cffi-1.14.6-cp35-cp35m-win32.whl", hash = "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca"}, - {file = "cffi-1.14.6-cp35-cp35m-win_amd64.whl", hash = "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218"}, {file = "cffi-1.14.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195"}, @@ -1409,8 +1470,8 @@ chardet = [ {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, - {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, + {file = "charset-normalizer-2.0.6.tar.gz", hash = "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"}, + {file = "charset_normalizer-2.0.6-py3-none-any.whl", hash = "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6"}, ] click = [ {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, @@ -1483,11 +1544,19 @@ coverage = [ {file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"}, {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] +desert = [ + {file = "desert-2020.11.18-py3-none-any.whl", hash = "sha256:6392702be7952fb9c8bbc775425fa929c1eab5c06552c2890e82a24964eeb084"}, + {file = "desert-2020.11.18.tar.gz", hash = "sha256:d7b7fb521dc84eec955a766ed7e37349f998cf047f37fd9596cb09737d63c62d"}, +] "discord.py" = [] distlib = [ {file = "distlib-0.3.2-py2.py3-none-any.whl", hash = "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c"}, {file = "distlib-0.3.2.zip", hash = "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736"}, ] +environs = [ + {file = "environs-9.3.3-py2.py3-none-any.whl", hash = "sha256:ee5466156b50fe03aa9fec6e720feea577b5bf515d7f21b2c46608272557ba26"}, + {file = "environs-9.3.3.tar.gz", hash = "sha256:72b867ff7b553076cdd90f3ee01ecc1cf854987639c9c459f0ed0d3d44ae490c"}, +] execnet = [ {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, @@ -1542,16 +1611,16 @@ gitdb = [ {file = "gitdb-4.0.7.tar.gz", hash = "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"}, ] gitpython = [ - {file = "GitPython-3.1.20-py3-none-any.whl", hash = "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a"}, - {file = "GitPython-3.1.20.tar.gz", hash = "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519"}, + {file = "GitPython-3.1.24-py3-none-any.whl", hash = "sha256:dc0a7f2f697657acc8d7f89033e8b1ea94dd90356b2983bca89dc8d2ab3cc647"}, + {file = "GitPython-3.1.24.tar.gz", hash = "sha256:df83fdf5e684fef7c6ee2c02fc68a5ceb7e7e759d08b694088d0cacb4eba59e5"}, ] humanfriendly = [ - {file = "humanfriendly-9.2-py2.py3-none-any.whl", hash = "sha256:332da98c24cc150efcc91b5508b19115209272bfdf4b0764a56795932f854271"}, - {file = "humanfriendly-9.2.tar.gz", hash = "sha256:f7dba53ac7935fd0b4a2fc9a29e316ddd9ea135fb3052d3d0279d10c18ff9c48"}, + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, ] identify = [ - {file = "identify-2.2.13-py2.py3-none-any.whl", hash = "sha256:7199679b5be13a6b40e6e19ea473e789b11b4e3b60986499b1f589ffb03c217c"}, - {file = "identify-2.2.13.tar.gz", hash = "sha256:7bc6e829392bd017236531963d2d937d66fc27cadc643ac0aba2ce9f26157c79"}, + {file = "identify-2.2.14-py2.py3-none-any.whl", hash = "sha256:113a76a6ba614d2a3dd408b3504446bcfac0370da5995aa6a17fd7c6dffde02d"}, + {file = "identify-2.2.14.tar.gz", hash = "sha256:32f465f3c48083f345ad29a9df8419a4ce0674bf4a8c3245191d65c83634bdbf"}, ] idna = [ {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, @@ -1578,12 +1647,22 @@ markdown = [ {file = "Markdown-3.3.4.tar.gz", hash = "sha256:31b5b491868dcc87d6c24b7e3d19a0d730d59d3e46f4eea6430a321bed387a49"}, ] markupsafe = [ + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, @@ -1592,14 +1671,21 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, @@ -1609,10 +1695,21 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] +marshmallow = [ + {file = "marshmallow-3.13.0-py2.py3-none-any.whl", hash = "sha256:dd4724335d3c2b870b641ffe4a2f8728a1380cd2e7e2312756715ffeaa82b842"}, + {file = "marshmallow-3.13.0.tar.gz", hash = "sha256:c67929438fd73a2be92128caa0325b1b5ed8b626d91a094d2f7f2771bf1f1c0e"}, +] +marshmallow-enum = [ + {file = "marshmallow-enum-1.5.1.tar.gz", hash = "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58"}, + {file = "marshmallow_enum-1.5.1-py2.py3-none-any.whl", hash = "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072"}, +] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, @@ -1634,8 +1731,8 @@ mkdocs-material = [ {file = "mkdocs_material-7.2.6-py2.py3-none-any.whl", hash = "sha256:4c6939b9d7d5c6db948ab02df8525c64211828ddf33286acea8b9d2115cec369"}, ] mkdocs-material-extensions = [ - {file = "mkdocs-material-extensions-1.0.1.tar.gz", hash = "sha256:6947fb7f5e4291e3c61405bad3539d81e0b3cd62ae0d66ced018128af509c68f"}, - {file = "mkdocs_material_extensions-1.0.1-py3-none-any.whl", hash = "sha256:d90c807a88348aa6d1805657ec5c0b2d8d609c110e62b9dce4daf7fa981fa338"}, + {file = "mkdocs-material-extensions-1.0.3.tar.gz", hash = "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2"}, + {file = "mkdocs_material_extensions-1.0.3-py3-none-any.whl", hash = "sha256:a82b70e533ce060b2a5d9eb2bc2e1be201cf61f901f93704b4acf6e3d5983a44"}, ] mslex = [ {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, @@ -1713,8 +1810,8 @@ pluggy = [ {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] pre-commit = [ - {file = "pre_commit-2.14.1-py2.py3-none-any.whl", hash = "sha256:a22d12a02da4d8df314187dfe7a61bda6291d57992060522feed30c8cd658b68"}, - {file = "pre_commit-2.14.1.tar.gz", hash = "sha256:7977a3103927932d4823178cbe4719ab55bb336f42a9f3bb2776cff99007a117"}, + {file = "pre_commit-2.15.0-py2.py3-none-any.whl", hash = "sha256:a4ed01000afcb484d9eb8d504272e642c4c4099bbad3a6b27e519bd6a3e928a6"}, + {file = "pre_commit-2.15.0.tar.gz", hash = "sha256:3c25add78dbdfb6a28a651780d5c311ac40dd17f160eb3954a0c59da40a505a7"}, ] psutil = [ {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, @@ -1793,30 +1890,6 @@ pycparser = [ {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, ] -pydantic = [ - {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, - {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, - {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, - {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, - {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, - {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, - {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, - {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, - {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, - {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, -] pydocstyle = [ {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, {file = "pydocstyle-6.1.1.tar.gz", hash = "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc"}, @@ -1837,10 +1910,9 @@ pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] -pyreadline = [ - {file = "pyreadline-2.1.win-amd64.exe", hash = "sha256:9ce5fa65b8992dfa373bddc5b6e0864ead8f291c94fbfec05fbd5c836162e67b"}, - {file = "pyreadline-2.1.win32.exe", hash = "sha256:65540c21bfe14405a3a77e4c085ecfce88724743a4ead47c66b84defcf82c32e"}, - {file = "pyreadline-2.1.zip", hash = "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1"}, +pyreadline3 = [ + {file = "pyreadline3-3.3-py3-none-any.whl", hash = "sha256:0003fd0079d152ecbd8111202c5a7dfa6a5569ffd65b235e45f3c2ecbee337b4"}, + {file = "pyreadline3-3.3.tar.gz", hash = "sha256:ff3b5a1ac0010d0967869f723e687d42cabc7dccf33b14934c92aa5168d260b3"}, ] pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, @@ -1975,8 +2047,8 @@ stevedore = [ {file = "stevedore-3.4.0.tar.gz", hash = "sha256:59b58edb7f57b11897f150475e7bc0c39c5381f0b8e3fa9f5c20ce6c89ec4aa1"}, ] taskipy = [ - {file = "taskipy-1.8.1-py3-none-any.whl", hash = "sha256:2b98f499966e40175d1f1306a64587f49dfa41b90d0d86c8f28b067cc58d0a56"}, - {file = "taskipy-1.8.1.tar.gz", hash = "sha256:7a2404125817e45d80e13fa663cae35da6e8ba590230094e815633653e25f98f"}, + {file = "taskipy-1.8.2-py3-none-any.whl", hash = "sha256:24b899ae17908fe9a61f4dc596792d5d2afef471ecfd8c6e9abb68a9568b40b7"}, + {file = "taskipy-1.8.2.tar.gz", hash = "sha256:36e958f646f2c435b39f748b8bbdb0b9c33a2c4a96b293427feaf48ad4e2caa0"}, ] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, @@ -1998,13 +2070,18 @@ typing-extensions = [ {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, ] +typing-inspect = [ + {file = "typing_inspect-0.7.1-py2-none-any.whl", hash = "sha256:b1f56c0783ef0f25fb064a01be6e5407e54cf4a4bf4f3ba3fe51e0bd6dcea9e5"}, + {file = "typing_inspect-0.7.1-py3-none-any.whl", hash = "sha256:3cd7d4563e997719a710a3bfe7ffb544c6b72069b6812a02e9b414a8fa3aaa6b"}, + {file = "typing_inspect-0.7.1.tar.gz", hash = "sha256:047d4097d9b17f46531bf6f014356111a1b6fb821a24fe7ac909853ca2a782aa"}, +] urllib3 = [ {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, ] virtualenv = [ - {file = "virtualenv-20.7.2-py2.py3-none-any.whl", hash = "sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06"}, - {file = "virtualenv-20.7.2.tar.gz", hash = "sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0"}, + {file = "virtualenv-20.8.0-py2.py3-none-any.whl", hash = "sha256:a4b987ec31c3c9996cf1bc865332f967fe4a0512c41b39652d6224f696e69da5"}, + {file = "virtualenv-20.8.0.tar.gz", hash = "sha256:4da4ac43888e97de9cf4fdd870f48ed864bbfd133d2c46cbdec941fed4a25aef"}, ] watchdog = [ {file = "watchdog-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5f57ce4f7e498278fb2a091f39359930144a0f2f90ea8cbf4523c4e25de34028"}, diff --git a/pyproject.toml b/pyproject.toml index e809f636..dad595b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,14 @@ arrow = "^1.1.1" colorama = "^0.4.3" coloredlogs = "^15.0" "discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/master.zip" } -pydantic = { version = "^1.8.2", extras = ["dotenv"] } -toml = "^0.10.2" +atoml = "^1.0.3" +attrs = "^21.2.0" +desert = "^2020.11.18" +environs = "~=9.3.3" +marshmallow = "~=3.13.0" +python-dotenv = "^0.19.0" +typing-extensions = "^3.10.0.2" +marshmallow-enum = "^1.5.1" [tool.poetry.extras] diff --git a/requirements.txt b/requirements.txt index 6ed382e0..3ffd847d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,24 +5,29 @@ aiodns==3.0.0; python_version >= "3.6" or python_version >= "3.6" and python_ful aiohttp==3.7.4.post0; python_version >= "3.6" arrow==1.1.1; python_version >= "3.6" async-timeout==3.0.1; python_full_version >= "3.5.3" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" +atoml==1.0.3; python_version >= "3.6" +attrs==21.2.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") brotlipy==0.7.0; python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" cchardet==2.1.7; python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" cffi==1.14.6; python_version >= "3.6" chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") coloredlogs==15.0.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +desert==2020.11.18; python_version >= "3.6" discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_full_version >= "3.8.0" -humanfriendly==9.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" +environs==9.3.3; python_version >= "3.6" +humanfriendly==10.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" idna==3.2; python_version >= "3.6" +marshmallow-enum==1.5.1 +marshmallow==3.13.0; python_version >= "3.5" multidict==5.1.0; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" +mypy-extensions==0.4.3; python_version >= "3.6" pycares==4.0.0; python_version >= "3.6" pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" -pydantic==1.8.2; python_full_version >= "3.6.1" -pyreadline==2.1; python_version >= "2.7" and python_full_version < "3.0.0" and sys_platform == "win32" or python_full_version >= "3.5.0" and sys_platform == "win32" +pyreadline3==3.3; sys_platform == "win32" and python_version >= "3.8" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0") python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -python-dotenv==0.19.0; python_full_version >= "3.6.1" and python_version >= "3.5" +python-dotenv==0.19.0; python_version >= "3.5" six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") -typing-extensions==3.10.0.0; python_version >= "3.6" or python_full_version >= "3.6.1" or python_version >= "3.6" and python_full_version >= "3.8.0" +typing-extensions==3.10.0.2 +typing-inspect==0.7.1; python_version >= "3.6" yarl==1.6.3; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" From 6a1b0c26a6ffab07cd97db5f48ec6e4b4d7ef310 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 01:51:32 -0400 Subject: [PATCH 02/76] config: remove config_default.toml since its now part of the classes Signed-off-by: onerandomusername --- modmail/config-default.toml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 modmail/config-default.toml diff --git a/modmail/config-default.toml b/modmail/config-default.toml deleted file mode 100644 index 29f7cfc5..00000000 --- a/modmail/config-default.toml +++ /dev/null @@ -1,10 +0,0 @@ -[bot] -prefix = "?" - -[colors] - -[dev] -log_level = 25 # "NOTICE" - -[dev.mode] -production = true From 5afc19691eb723b3cf1d96b84c63fd94b1cfe1d5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 01:52:43 -0400 Subject: [PATCH 03/76] fix: add missing items to the configuration Signed-off-by: onerandomusername --- modmail/__init__.py | 2 +- modmail/config.py | 128 +++++++++++++++++++++++++++++++------------- tox.ini | 1 - 3 files changed, 91 insertions(+), 40 deletions(-) diff --git a/modmail/__init__.py b/modmail/__init__.py index 8f61401e..0d07a6e8 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -9,7 +9,7 @@ env = environs.Env() -env.read_env(".env") +env.read_env(".env", recurse=False) logging.TRACE = 5 logging.NOTICE = 25 diff --git a/modmail/config.py b/modmail/config.py index 9dbd1396..b3f8e096 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -2,44 +2,75 @@ import logging import os import pathlib -import sys import typing from collections import defaultdict -from pprint import pprint import atoml import attr import desert import discord +import discord.ext.commands.converter import environs import marshmallow import marshmallow.fields import marshmallow.validate +_CWD = pathlib.Path(os.getcwd()) ENV_PREFIX = "MODMAIL_" +USER_CONFIG_TOML = _CWD / "modmail_config.toml" env = environs.Env(eager=False, expand_vars=True) +env.read_env(_CWD / ".env", recurse=False) -DEFAULT_CONFIG_PATH = pathlib.Path(os.path.join(os.path.dirname(__file__), "test.toml")) - - -def _generate_default_dict(): +def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" return defaultdict(_generate_default_dict) +unparsed_user_provided_cfg = defaultdict(lambda: marshmallow.missing) try: - with open(DEFAULT_CONFIG_PATH) as f: - unparsed_user_provided_cfg = defaultdict(_generate_default_dict, atoml.parse(f.read()).value) + with open(USER_CONFIG_TOML) as f: + unparsed_user_provided_cfg.update(atoml.parse(f.read()).value) except FileNotFoundError: - unparsed_user_provided_cfg = defaultdict(_generate_default_dict) + pass + + +def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: + """Convert a string or integer to a discord.Colour. Also supports being passed a discord.Colour.""" + if isinstance(col, discord.Colour): + return col + if isinstance(col, str): + col = int(col, 16) + return discord.Colour(col) class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deseriaze into a string.""" + class ColourConvert(discord.ext.commands.converter.ColourConverter): + def convert(self, argument: str) -> discord.Colour: + if argument[0] == "#": + return self.parse_hex_number(argument[1:]) + + if argument[0:2] == "0x": + rest = argument[2:] + # Legacy backwards compatible syntax + if rest.startswith("#"): + return self.parse_hex_number(rest[1:]) + return self.parse_hex_number(rest) + + arg = argument.lower() + if arg[0:3] == "rgb": + return self.parse_rgb(arg) + + arg = arg.replace(" ", "_") + method = getattr(discord.Colour, arg, None) + if arg.startswith("from_") or method is None or not inspect.ismethod(method): + raise discord.ext.commands.converter.BadColourArgument(arg) + return method() + def _serialize(self, value: typing.Any, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: return str(value.value) @@ -51,16 +82,17 @@ def _deserialize( **kwargs, ) -> str: if not isinstance(value, discord.Colour): - value = discord.Colour(int(value, 16)) + if isinstance(value, str): + value = self.ColourConvert().convert(value) + else: + value = discord.Colour(value) return value -# marshmallow.fields.Colour = _ColourField - - -@attr.s(auto_attribs=True, frozen=True) +@attr.s(auto_attribs=True) class Bot: - """Values that are configuration for the bot itself. + """ + Values that are configuration for the bot itself. These are metavalues, and are the token, prefix, database bind, basically all of the stuff that needs to be known BEFORE attempting to log in to the database or discord. @@ -68,6 +100,7 @@ class Bot: token: str = attr.ib( default=marshmallow.missing, + on_setattr=attr.setters.NO_OP, metadata={ "required": True, "load_only": True, @@ -76,28 +109,34 @@ class Bot: ) prefix: str = attr.ib( default="?", + converter=lambda x: "?" if x is None else x, metadata={ "allow_none": False, }, ) class Meta: + """Marshmallow configuration class for schema generation.""" + load_only = ("token",) partial = True -def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: - if isinstance(col, discord.Colour): - return col - if isinstance(col, str): - col = int(col, 16) - return discord.Colour(col) +@attr.s(auto_attribs=True, frozen=True) +class BotModeCfg: + """ + The three bot modes for the bot. Enabling some of these may enable other bot features. + `production` is used internally and is always True. + `develop` enables additonal features which are useful for bot developers. + `plugin_dev` enables additional commands which are useful when working with plugins. + """ -@attr.s(auto_attribs=True) -class BotModeCfg: production: bool = desert.ib( - marshmallow.fields.Constant(True), default=True, metadata={"dump_default": True, "dump_only": True} + marshmallow.fields.Constant(True), + default=True, + converter=lambda _: True, + metadata={"dump_default": True, "dump_only": True}, ) develop: bool = attr.ib(default=False, metadata={"allow_none": False}) plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) @@ -118,6 +157,8 @@ class Colours: @attr.s(auto_attribs=True) class DevCfg: + """Developer configuration. These values should not be changed unless you know what you're doing.""" + mode: BotModeCfg = BotModeCfg() log_level: int = desert.ib( marshmallow.fields.Integer( @@ -129,11 +170,20 @@ class DevCfg: @attr.s(auto_attribs=True) class Cfg: + """ + Base configuration attrs class. + + The reason this creates defaults of the variables is so + we can get a clean default variable if we don't pass anything. + """ + bot: Bot = Bot() colours: Colours = Colours() dev: DevCfg = DevCfg() class Meta: + """Marshmallow configuration class for schema generation.""" + exclude = ("bot.token",) @@ -165,30 +215,32 @@ def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing return klass(**kw) +_toml_bot_cfg = unparsed_user_provided_cfg["bot"] unparsed_user_provided_cfg["bot"] = attr.asdict( - _build_bot_class(Bot, "BOT_", unparsed_user_provided_cfg["bot"]) + _build_bot_class(Bot, "BOT_", _toml_bot_cfg if _toml_bot_cfg is not marshmallow.missing else None) ) - -# env.seal() - +del _toml_bot_cfg # build configuration Configuration = desert.schema_class(Cfg) # noqa: N818 -USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.RAISE) - +USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.EXCLUDE) +DEFAULTS_CONFIG = Cfg() -# hide the bot token from serilzation -# this prevents the token from being saved to places. +@attr.s(auto_attribs=True, slots=True, kw_only=True) +class Config: + """ + Base configuration variable. Used across the entire bot for configuration variables. -DEFAULTS_CONFIG = Cfg() + Holds two variables, default and user. + Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + User is a Cfg schema instance, generated from a combination of the defaults, + user provided toml, and environment variables. + """ -@attr.s(auto_attribs=True, slots=True) -class Config: user: Cfg default: Cfg + schema: marshmallow.Schema -config = Config(USER_PROVIDED_CONFIG, DEFAULTS_CONFIG) - -print(config.user.bot.prefix) +config = Config(user=USER_PROVIDED_CONFIG, default=DEFAULTS_CONFIG, schema=Configuration) diff --git a/tox.ini b/tox.ini index bc557a04..405a6e64 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,6 @@ max-line-length=110 application_import_names=modmail docstring-convention=all exclude= - modmail/config.py, __pycache__, .cache,.git, .md,.svg,.png, From f8fabf5f034acb31d6f5986f475ec54de47dac4c Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 03:23:37 -0400 Subject: [PATCH 04/76] fix: have a test token in the pytest env Signed-off-by: onerandomusername --- .github/workflows/lint_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index a9bb18c5..293d1941 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -162,6 +162,8 @@ jobs: # coverage report to github. - name: Run tests and generate coverage report run: pytest -n auto --dist loadfile --cov --disable-warnings -q + env: + MODMAIL_BOT_TOKEN: NjQzOTQ1MjY0ODY4MDk4MDQ5.342b.4inDLBILY69LOLfyi6jk420dpyjoEVsCoModM # This step will publish the coverage reports to codecov.io and # print a "job" link in the output of the GitHub Action From 6cc4e20bce8954c4ff45ea6f66b41e134e1164f7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 13:24:18 -0400 Subject: [PATCH 05/76] enable slots on cfg classes, fix colour converter, properly prevent token from being changed Signed-off-by: onerandomusername --- modmail/config.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index b3f8e096..4fa0938c 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -37,20 +37,16 @@ def _generate_default_dict() -> defaultdict: pass -def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: - """Convert a string or integer to a discord.Colour. Also supports being passed a discord.Colour.""" - if isinstance(col, discord.Colour): - return col - if isinstance(col, str): - col = int(col, 16) - return discord.Colour(col) - - class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deseriaze into a string.""" class ColourConvert(discord.ext.commands.converter.ColourConverter): def convert(self, argument: str) -> discord.Colour: + if isinstance(argument, discord.Colour): + return argument + if not isinstance(argument, str): + argument = str(argument) + if argument[0] == "#": return self.parse_hex_number(argument[1:]) @@ -89,7 +85,12 @@ def _deserialize( return value -@attr.s(auto_attribs=True) +def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Colour: + """Convert a string or integer to a discord.Colour. Also supports being passed a discord.Colour.""" + return _ColourField.ColourConvert().convert(col) + + +@attr.s(auto_attribs=True, slots=True) class Bot: """ Values that are configuration for the bot itself. @@ -100,10 +101,11 @@ class Bot: token: str = attr.ib( default=marshmallow.missing, - on_setattr=attr.setters.NO_OP, + on_setattr=attr.setters.frozen, + repr=False, metadata={ "required": True, - "load_only": True, + "dump_only": True, "allow_none": False, }, ) @@ -122,7 +124,7 @@ class Meta: partial = True -@attr.s(auto_attribs=True, frozen=True) +@attr.s(auto_attribs=True, slots=True, frozen=True) class BotModeCfg: """ The three bot modes for the bot. Enabling some of these may enable other bot features. @@ -142,7 +144,7 @@ class BotModeCfg: plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) -@attr.s(auto_attribs=True) +@attr.s(auto_attribs=True, slots=True) class Colours: """ Default colors. @@ -155,20 +157,21 @@ class Colours: ) -@attr.s(auto_attribs=True) +@attr.s(auto_attribs=True, slots=True) class DevCfg: """Developer configuration. These values should not be changed unless you know what you're doing.""" mode: BotModeCfg = BotModeCfg() - log_level: int = desert.ib( - marshmallow.fields.Integer( - validate=marshmallow.validate.Range(0, 50, error="Logging level must be within 0 to 50.") - ), - default=logging.INFO, - ) + log_level: int = attr.ib(default=logging.INFO) + + @log_level.validator + def _log_level_validator(self, _: attr.Attribute, value: int) -> None: + """Validate that log_level is within 0 to 50.""" + if value not in range(0, 50): + raise ValueError("log_level must be an integer within 0 to 50.") -@attr.s(auto_attribs=True) +@attr.s(auto_attribs=True, slots=True) class Cfg: """ Base configuration attrs class. From 82b1c73221bf55bdacc1ed1245fd3c1d5dd4be60 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 19:26:31 -0400 Subject: [PATCH 06/76] fix: remove unused class definitions when converting to a marshmallow schema, desert does not use class defined Meta classes Signed-off-by: onerandomusername --- modmail/config.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 4fa0938c..f482411f 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -117,12 +117,6 @@ class Bot: }, ) - class Meta: - """Marshmallow configuration class for schema generation.""" - - load_only = ("token",) - partial = True - @attr.s(auto_attribs=True, slots=True, frozen=True) class BotModeCfg: @@ -184,11 +178,6 @@ class Cfg: colours: Colours = Colours() dev: DevCfg = DevCfg() - class Meta: - """Marshmallow configuration class for schema generation.""" - - exclude = ("bot.token",) - # find and build a bot class from our env def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing.Dict = None) -> Bot: @@ -224,7 +213,7 @@ def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing ) del _toml_bot_cfg # build configuration -Configuration = desert.schema_class(Cfg) # noqa: N818 +Configuration = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.EXCLUDE) DEFAULTS_CONFIG = Cfg() From 4a80406ae833eb83f18dc7f8cc5f35570ea0a26e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 19:28:04 -0400 Subject: [PATCH 07/76] config: add autogenerated default cfg toml file Signed-off-by: onerandomusername --- .gitignore | 2 +- modmail/config.py | 31 +++++++++++++++++++++++++++++++ modmail/default_config.toml | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 modmail/default_config.toml diff --git a/.gitignore b/.gitignore index 523110e0..d737f24f 100644 --- a/.gitignore +++ b/.gitignore @@ -136,7 +136,7 @@ dmypy.json logs # Configuration -*config.toml +/modmail_config.toml # Custom docker compose override docker-compose.override.yml diff --git a/modmail/config.py b/modmail/config.py index f482411f..49e88dcc 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -236,3 +236,34 @@ class Config: config = Config(user=USER_PROVIDED_CONFIG, default=DEFAULTS_CONFIG, schema=Configuration) + +if __name__ == "__main__": + # if this file is run directly, export the configuration to a defaults file. + + dump: dict = Configuration().dump(DEFAULTS_CONFIG) + + # Sort the dictionary configuration. + # This is the only place where the order of the config should matter, when exporting in a specific style + def sort_dict(d: dict) -> dict: + """Takes a dict and sorts it, recursively.""" + sorted_dict = {x[0]: x[1] for x in sorted(d.items(), key=lambda e: e[0])} + + for k, v in d.items(): + if not isinstance(v, dict): + continue + sorted_dict[k] = sort_dict(v) + + return sorted_dict + + dump = sort_dict(dump) + + doc = atoml.document() + doc.add(atoml.comment("This is an autogenerated TOML document.")) + doc.add(atoml.comment("Directly run the config.py file to generate.")) + doc.add(atoml.nl()) + + doc.update(dump) + + # toml + with open(pathlib.Path(__file__).parent / "default_config.toml", "w") as f: + atoml.dump(doc, f) diff --git a/modmail/default_config.toml b/modmail/default_config.toml new file mode 100644 index 00000000..a4283f64 --- /dev/null +++ b/modmail/default_config.toml @@ -0,0 +1,18 @@ +# This is an autogenerated TOML document. +# Directly run the config.py file to generate. + +answer = 42 + +[bot] +prefix = "?" + +[colours] +base_embed_color = "7506394" + +[dev] +log_level = 20 + +[dev.mode] +develop = false +plugin_dev = false +production = true From 311ff5445c94fa145c0db18931194cd7ed36953f Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 20:44:10 -0400 Subject: [PATCH 08/76] fix: properly deserialize discord.Colour to a string representation Signed-off-by: onerandomusername --- modmail/config.py | 21 +++++++++++++-------- modmail/default_config.toml | 3 +-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 49e88dcc..91c27e82 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -38,10 +38,18 @@ def _generate_default_dict() -> defaultdict: class _ColourField(marshmallow.fields.Field): - """Class to convert a str or int into a color and deseriaze into a string.""" + """Class to convert a str or int into a color and deserialize into a string.""" class ColourConvert(discord.ext.commands.converter.ColourConverter): - def convert(self, argument: str) -> discord.Colour: + """Inherited discord.py colour converter.""" + + def convert(self, argument: typing.Union[str, int, discord.Colour]) -> discord.Colour: + """ + Convert an argument into a discord.Colour. + + This code was copied from discord.ext.commands.converter.ColourConverter. + Modified to not be async or need a context since it was not used in the first place. + """ if isinstance(argument, discord.Colour): return argument if not isinstance(argument, str): @@ -67,8 +75,8 @@ def convert(self, argument: str) -> discord.Colour: raise discord.ext.commands.converter.BadColourArgument(arg) return method() - def _serialize(self, value: typing.Any, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: - return str(value.value) + def _serialize(self, value: discord.Colour, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: + return "#" + hex(value.value)[2:].lower() def _deserialize( self, @@ -78,10 +86,7 @@ def _deserialize( **kwargs, ) -> str: if not isinstance(value, discord.Colour): - if isinstance(value, str): - value = self.ColourConvert().convert(value) - else: - value = discord.Colour(value) + value = self.ColourConvert().convert(value) return value diff --git a/modmail/default_config.toml b/modmail/default_config.toml index a4283f64..935be134 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,13 +1,12 @@ # This is an autogenerated TOML document. # Directly run the config.py file to generate. -answer = 42 [bot] prefix = "?" [colours] -base_embed_color = "7506394" +base_embed_color = "#7289da" [dev] log_level = 20 From 62aa7dcb39164652d39c8ace6850ed4a3a655815 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 20:48:29 -0400 Subject: [PATCH 09/76] feat: add configuration via a yaml cfg file Signed-off-by: onerandomusername --- .gitignore | 3 ++- modmail/config.py | 27 +++++++++++++++++++++------ modmail/default_config.yaml | 10 ++++++++++ poetry.lock | 9 ++++++--- pyproject.toml | 3 ++- 5 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 modmail/default_config.yaml diff --git a/.gitignore b/.gitignore index d737f24f..d52be37a 100644 --- a/.gitignore +++ b/.gitignore @@ -133,10 +133,11 @@ dmypy.json .idea/ # logs -logs +/logs/ # Configuration /modmail_config.toml +/modmail_config.yaml # Custom docker compose override docker-compose.override.yml diff --git a/modmail/config.py b/modmail/config.py index 91c27e82..98f14c7d 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,6 +1,5 @@ import inspect import logging -import os import pathlib import typing from collections import defaultdict @@ -16,12 +15,12 @@ import marshmallow.validate -_CWD = pathlib.Path(os.getcwd()) ENV_PREFIX = "MODMAIL_" -USER_CONFIG_TOML = _CWD / "modmail_config.toml" +USER_CONFIG_FILE_NAME = "modmail_config" +AUTO_GEN_FILE_NAME = "default_config" env = environs.Env(eager=False, expand_vars=True) -env.read_env(_CWD / ".env", recurse=False) +env.read_env(pathlib.Path.cwd() / ".env", recurse=False) def _generate_default_dict() -> defaultdict: @@ -31,11 +30,23 @@ def _generate_default_dict() -> defaultdict: unparsed_user_provided_cfg = defaultdict(lambda: marshmallow.missing) try: - with open(USER_CONFIG_TOML) as f: + with open(pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".toml")) as f: unparsed_user_provided_cfg.update(atoml.parse(f.read()).value) except FileNotFoundError: pass +# attempt to also check for a yaml file +try: + import yaml +except ImportError: + pass +else: + # check for an existing file + yaml_cfg = pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".yaml") + if yaml_cfg.is_file(): + with open(yaml_cfg, "r") as f: + unparsed_user_provided_cfg.update(yaml.load(f.read(), Loader=yaml.SafeLoader)) + class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deserialize into a string.""" @@ -270,5 +281,9 @@ def sort_dict(d: dict) -> dict: doc.update(dump) # toml - with open(pathlib.Path(__file__).parent / "default_config.toml", "w") as f: + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: atoml.dump(doc, f) + + # yaml + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: + yaml.dump(dump, f, indent=4) diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml new file mode 100644 index 00000000..c05fd99e --- /dev/null +++ b/modmail/default_config.yaml @@ -0,0 +1,10 @@ +bot: + prefix: '?' +colours: + base_embed_color: '#7289da' +dev: + log_level: 20 + mode: + develop: false + plugin_dev: false + production: true diff --git a/poetry.lock b/poetry.lock index 86aa99a8..07a497a0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1050,8 +1050,8 @@ cli = ["click (>=5.0)"] name = "pyyaml" version = "5.4.1" description = "YAML parser and emitter for Python" -category = "dev" -optional = false +category = "main" +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [[package]] @@ -1264,10 +1264,13 @@ python-versions = ">=3.6" docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +[extras] +yaml = ["PyYAML"] + [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "6cfec48a5437401e5e5aeece5efc90f48eae9e2903d6117c4f2cc850af33cdab" +content-hash = "74f9246891ee29ca657788bf12759a32664eb024b7e194ca214d19ed8c6784c5" [metadata.files] aiodns = [ diff --git a/pyproject.toml b/pyproject.toml index dad595b0..96395a3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,12 +25,13 @@ desert = "^2020.11.18" environs = "~=9.3.3" marshmallow = "~=3.13.0" python-dotenv = "^0.19.0" +PyYAML = { version = "^5.4.1", optional = true } typing-extensions = "^3.10.0.2" marshmallow-enum = "^1.5.1" [tool.poetry.extras] - +yaml = ["pyyaml"] [tool.poetry.dev-dependencies] # always needed From e4d9e1ed490f2cd740959f4f88203188072da6b7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 19 Sep 2021 21:36:16 -0400 Subject: [PATCH 10/76] fix: allow log level 50, as documented Signed-off-by: onerandomusername --- modmail/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 98f14c7d..2acdaee6 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -177,8 +177,8 @@ class DevCfg: @log_level.validator def _log_level_validator(self, _: attr.Attribute, value: int) -> None: """Validate that log_level is within 0 to 50.""" - if value not in range(0, 50): - raise ValueError("log_level must be an integer within 0 to 50.") + if value not in range(0, 50 + 1): + raise ValueError("log_level must be an integer within 0 to 50, inclusive.") @attr.s(auto_attribs=True, slots=True) From 70128af93392ae316d160bc9cc2cd98031afaea7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 00:29:12 -0400 Subject: [PATCH 11/76] feat: made testable config system by refactor config file loading Signed-off-by: onerandomusername --- modmail/config.py | 276 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 228 insertions(+), 48 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 2acdaee6..6590f840 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,5 +1,6 @@ import inspect import logging +import os import pathlib import typing from collections import defaultdict @@ -15,12 +16,42 @@ import marshmallow.validate +try: + import yaml +except ImportError: + yaml = None + +__all__ = [ + "AUTO_GEN_FILE_NAME", + "DEFAULT_CONFIG_FILES", + "ENV_PREFIX", + "USER_CONFIG_FILE_NAME", + "CfgLoadError", + "Config", + "config", + "ConfigurationSchema", + "Bot", + "BotModeCfg", + "Cfg", + "Colours", + "DevCfg", + "convert_to_color", + "export_default_conf", + "get_config", + "get_default_config", + "load_toml", + "load_yaml", +] + +_CWD = pathlib.Path.cwd() + ENV_PREFIX = "MODMAIL_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" - -env = environs.Env(eager=False, expand_vars=True) -env.read_env(pathlib.Path.cwd() / ".env", recurse=False) +DEFAULT_CONFIG_FILES = [ + _CWD / (USER_CONFIG_FILE_NAME + ".yaml"), + _CWD / (USER_CONFIG_FILE_NAME + ".toml"), +] def _generate_default_dict() -> defaultdict: @@ -28,24 +59,10 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -unparsed_user_provided_cfg = defaultdict(lambda: marshmallow.missing) -try: - with open(pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".toml")) as f: - unparsed_user_provided_cfg.update(atoml.parse(f.read()).value) -except FileNotFoundError: - pass +class CfgLoadError(Exception): + """Exception if the configuration failed to load from a local file.""" -# attempt to also check for a yaml file -try: - import yaml -except ImportError: - pass -else: - # check for an existing file - yaml_cfg = pathlib.Path.cwd() / (USER_CONFIG_FILE_NAME + ".yaml") - if yaml_cfg.is_file(): - with open(yaml_cfg, "r") as f: - unparsed_user_provided_cfg.update(yaml.load(f.read(), Loader=yaml.SafeLoader)) + ... class _ColourField(marshmallow.fields.Field): @@ -195,8 +212,34 @@ class Cfg: dev: DevCfg = DevCfg() +# build configuration +ConfigurationSchema = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 + + +_CACHED_CONFIG: "Config" = None + + +@attr.s(auto_attribs=True, slots=True, kw_only=True) +class Config: + """ + Base configuration variable. Used across the entire bot for configuration variables. + + Holds two variables, default and user. + Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + + User is a Cfg schema instance, generated from a combination of the defaults, + user provided toml, and environment variables. + """ + + user: Cfg + schema: marshmallow.Schema + default: Cfg = Cfg() + + # find and build a bot class from our env -def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing.Dict = None) -> Bot: +def _build_bot_class( + klass: typing.Any, env: environs.Env, class_prefix: str = "", defaults: typing.Dict = None +) -> Bot: """ Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. @@ -223,40 +266,159 @@ def _build_bot_class(klass: typing.Any, class_prefix: str = "", defaults: typing return klass(**kw) -_toml_bot_cfg = unparsed_user_provided_cfg["bot"] -unparsed_user_provided_cfg["bot"] = attr.asdict( - _build_bot_class(Bot, "BOT_", _toml_bot_cfg if _toml_bot_cfg is not marshmallow.missing else None) -) -del _toml_bot_cfg -# build configuration -Configuration = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 -USER_PROVIDED_CONFIG: Cfg = Configuration().load(unparsed_user_provided_cfg, unknown=marshmallow.EXCLUDE) -DEFAULTS_CONFIG = Cfg() +def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: + """ + Load a configuration dictionary from the specified env file and environment variables. + All dependencies for this will always be installed. + """ + if env_file is None: + env_file = _CWD / ".env" + else: + env_file = pathlib.Path(env_file) -@attr.s(auto_attribs=True, slots=True, kw_only=True) -class Config: + env = environs.Env(eager=False, expand_vars=True) + env.read_env(".env", recurse=False) + + if not existing_cfg_dict: + existing_cfg_dict = defaultdict(_generate_default_dict) + + existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, "BOT_"))) + + return existing_cfg_dict + + +def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defaultdict: """ - Base configuration variable. Used across the entire bot for configuration variables. + Load a configuration dictionary from the specified toml file. - Holds two variables, default and user. - Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + All dependencies for this will always be installed. + """ + if path is None: + path = (_CWD / (USER_CONFIG_FILE_NAME + ".toml"),) + else: + # fully resolve path + path = pathlib.Path(path) - User is a Cfg schema instance, generated from a combination of the defaults, - user provided toml, and environment variables. + if not path.is_file(): + raise CfgLoadError("The provided toml file path is not a valid file.") + + try: + with open(path) as f: + loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) + if existing_cfg_dict is not None: + loaded_cfg.update(existing_cfg_dict) + return existing_cfg_dict + else: + return loaded_cfg + except Exception as e: + raise CfgLoadError from e + + +def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: """ + Load a configuration dictionary from the specified yaml file. - user: Cfg - default: Cfg - schema: marshmallow.Schema + The dependency for this may not be installed, as toml is already used elsewhere, so this is optional. + + In order to keep errors at a minimum, this function checks if both pyyaml is installed + and if a yaml configuration file exists before raising an exception. + """ + if path is None: + path = (_CWD / (USER_CONFIG_FILE_NAME + ".yaml"),) + else: + path = pathlib.Path(path) + + states = [ + ("The yaml library is not installed.", yaml is not None), + ("The provided yaml config path does not exist.", path.exists()), + ("The provided yaml config file is not a readable file.", path.is_file()), + ] + if errors := "\n".join(msg for msg, check in states if not check): + raise CfgLoadError(errors) + + try: + with open(path, "r") as f: + loaded_cfg = dict(yaml.load(f.read(), Loader=yaml.SafeLoader)) + if existing_cfg_dict is not None: + loaded_cfg.update(existing_cfg_dict) + return existing_cfg_dict + else: + return loaded_cfg + except Exception as e: + raise CfgLoadError from e + + +def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: bool = True) -> Config: + """ + Loads a configuration from the specified files. + Configuration will stop loading on the first existing file. + Default order checks yaml, then toml. -config = Config(user=USER_PROVIDED_CONFIG, default=DEFAULTS_CONFIG, schema=Configuration) + Supported file types are .toml or .yaml + """ + # load the env first + if load_env: + env_cfg = _load_env() + else: + env_cfg = None + + if files is None: + files = DEFAULT_CONFIG_FILES + elif len(files) == 0: + raise CfgLoadError("At least one file to load from must be provided.") + + loaded_config_dict: dict = None + for file in files: + if not isinstance(file, pathlib.Path): + file = pathlib.Path(file) + if not file.exists(): + # file does not exist + continue + + if file.suffix == ".toml" and atoml is not None: + loaded_config_dict = load_toml(file, existing_cfg_dict=env_cfg) + break + elif file.suffix == ".yaml" and yaml is not None: + loaded_config_dict = load_yaml(file, existing_cfg_dict=env_cfg) + break + else: + raise Exception( + "Provided configuration file is not of a supported type or " + "the required dependencies are not installed." + ) + + if loaded_config_dict is None: + raise CfgLoadError( + "Not gonna lie, this SHOULD be unreachable...\n" + "If you came across this as a consumer, please report this bug to our bug tracker." + ) + loaded_config_dict = ConfigurationSchema().load(loaded_config_dict) + return Config(user=loaded_config_dict, schema=ConfigurationSchema) + + +def get_config() -> Config: + """ + Helps to try to ensure that only one instance of the Config class exists. + + This means that all usage of the configuration is using the same configuration class. + """ + global _CACHED_CONFIG + if _CACHED_CONFIG is None: + _CACHED_CONFIG = _load_config() + return _CACHED_CONFIG -if __name__ == "__main__": - # if this file is run directly, export the configuration to a defaults file. - dump: dict = Configuration().dump(DEFAULTS_CONFIG) +def get_default_config() -> Cfg: + """Get the default configuration instance of the global Config instance.""" + return get_config().default + + +def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> None: + """Export default configuration to the preconfigured locations.""" + conf = get_default_config() + dump: dict = ConfigurationSchema().dump(conf) # Sort the dictionary configuration. # This is the only place where the order of the config should matter, when exporting in a specific style @@ -281,9 +443,27 @@ def sort_dict(d: dict) -> dict: doc.update(dump) # toml - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: - atoml.dump(doc, f) + if export_toml: + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: + atoml.dump(doc, f) # yaml - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: - yaml.dump(dump, f, indent=4) + if export_yaml is True or (yaml is not None and export_yaml is None): + with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: + try: + yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + except AttributeError: + raise CfgLoadError( + "Tried to export the yaml configuration file but pyyaml is not installed." + ) from None + + +config = get_config() + + +if __name__ == "__main__": + # if this file is run directly, export the configuration to a defaults file. + import sys + + print("Exporting default configuration to desiginated default config files.") + sys.exit(export_default_conf()) From 4707d6ce3512a86f08b603c650d7340c2f27e95e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 00:50:26 -0400 Subject: [PATCH 12/76] tools: add pre-commit hook to automatically export default config if any modmail configuration file is edited, a pre-commit hook will run to check that the autogenerated files are up-to-date Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 10 ++++++++++ modmail/config.py | 10 +--------- scripts/__init__.py | 0 scripts/export_new_config_to_default_config.py | 16 ++++++++++++++++ 4 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/export_new_config_to_default_config.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e826392..cff28bbe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,16 @@ ## Pre-commit setup repos: + # its possible to put this at the bottom, but we want the pre-commit-hooks to check these files as well. + - repo: local + hooks: + - id: ensure-default-configuration-is-exported + name: Export default configuration + language: python + entry: poetry run python -m scripts.export_new_config_to_default_config + files: '(modmail\/(config.py|default_config(.toml|.yaml)))$' + additional_dependencies: [pyyaml==5.4.1] + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: diff --git a/modmail/config.py b/modmail/config.py index 6590f840..da169238 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -415,7 +415,7 @@ def get_default_config() -> Cfg: return get_config().default -def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> None: +def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> bool: """Export default configuration to the preconfigured locations.""" conf = get_default_config() dump: dict = ConfigurationSchema().dump(conf) @@ -459,11 +459,3 @@ def sort_dict(d: dict) -> dict: config = get_config() - - -if __name__ == "__main__": - # if this file is run directly, export the configuration to a defaults file. - import sys - - print("Exporting default configuration to desiginated default config files.") - sys.exit(export_default_conf()) diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py new file mode 100644 index 00000000..81102ccb --- /dev/null +++ b/scripts/export_new_config_to_default_config.py @@ -0,0 +1,16 @@ +""" +Exports the configuration to the configuration default files. + +This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. +""" +import sys + +import atoml # noqa: F401 +import yaml # noqa: F401 + +import modmail.config + + +if __name__ == "__main__": + print("Exporting configuration to default files. If they exist, overwriting their contents.") + sys.exit(modmail.config.export_default_conf(export_yaml=True)) From 26cc817f7f3998f5ba031b145a4abb6dcf43e5ee Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 01:47:31 -0400 Subject: [PATCH 13/76] tools: automatically export required environment varaibles to .env.template Signed-off-by: onerandomusername --- .env.template | 2 +- .pre-commit-config.yaml | 6 +- modmail/config.py | 49 +--------- modmail/default_config.toml | 2 +- modmail/default_config.yaml | 2 + .../export_new_config_to_default_config.py | 96 ++++++++++++++++++- 6 files changed, 105 insertions(+), 52 deletions(-) diff --git a/.env.template b/.env.template index 79023bad..66c582eb 100644 --- a/.env.template +++ b/.env.template @@ -1 +1 @@ -TOKEN="MyBotToken" +MODMAIL_BOT_TOKEN="MyBotToken" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cff28bbe..654e7dae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,8 +8,10 @@ repos: name: Export default configuration language: python entry: poetry run python -m scripts.export_new_config_to_default_config - files: '(modmail\/(config.py|default_config(.toml|.yaml)))$' - additional_dependencies: [pyyaml==5.4.1] + files: '(\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + additional_dependencies: + - atoml~=1.0.3 + - pyyaml~=5.4.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 diff --git a/modmail/config.py b/modmail/config.py index da169238..db95793b 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -25,6 +25,7 @@ "AUTO_GEN_FILE_NAME", "DEFAULT_CONFIG_FILES", "ENV_PREFIX", + "BOT_ENV_PREFIX", "USER_CONFIG_FILE_NAME", "CfgLoadError", "Config", @@ -36,7 +37,6 @@ "Colours", "DevCfg", "convert_to_color", - "export_default_conf", "get_config", "get_default_config", "load_toml", @@ -46,6 +46,7 @@ _CWD = pathlib.Path.cwd() ENV_PREFIX = "MODMAIL_" +BOT_ENV_PREFIX = "BOT_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" DEFAULT_CONFIG_FILES = [ @@ -140,6 +141,7 @@ class Bot: "required": True, "dump_only": True, "allow_none": False, + "modmail_export_filler": "MyBotToken", }, ) prefix: str = attr.ib( @@ -283,7 +285,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, "BOT_"))) + existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX))) return existing_cfg_dict @@ -415,47 +417,4 @@ def get_default_config() -> Cfg: return get_config().default -def export_default_conf(*, export_toml: bool = True, export_yaml: bool = None) -> bool: - """Export default configuration to the preconfigured locations.""" - conf = get_default_config() - dump: dict = ConfigurationSchema().dump(conf) - - # Sort the dictionary configuration. - # This is the only place where the order of the config should matter, when exporting in a specific style - def sort_dict(d: dict) -> dict: - """Takes a dict and sorts it, recursively.""" - sorted_dict = {x[0]: x[1] for x in sorted(d.items(), key=lambda e: e[0])} - - for k, v in d.items(): - if not isinstance(v, dict): - continue - sorted_dict[k] = sort_dict(v) - - return sorted_dict - - dump = sort_dict(dump) - - doc = atoml.document() - doc.add(atoml.comment("This is an autogenerated TOML document.")) - doc.add(atoml.comment("Directly run the config.py file to generate.")) - doc.add(atoml.nl()) - - doc.update(dump) - - # toml - if export_toml: - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".toml"), "w") as f: - atoml.dump(doc, f) - - # yaml - if export_yaml is True or (yaml is not None and export_yaml is None): - with open(pathlib.Path(__file__).parent / (AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: - try: - yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) - except AttributeError: - raise CfgLoadError( - "Tried to export the yaml configuration file but pyyaml is not installed." - ) from None - - config = get_config() diff --git a/modmail/default_config.toml b/modmail/default_config.toml index 935be134..de41e516 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,5 +1,5 @@ # This is an autogenerated TOML document. -# Directly run the config.py file to generate. +# Directly run scripts/export_new_config_to_default_config.py to generate. [bot] diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index c05fd99e..7ada283b 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -1,3 +1,5 @@ +# This is an autogenerated YAML document. +# Directly run scripts/export_new_config_to_default_config.py to generate. bot: prefix: '?' colours: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 81102ccb..26aba0ef 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -3,14 +3,104 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ +import pathlib import sys +from collections import defaultdict -import atoml # noqa: F401 -import yaml # noqa: F401 +import atoml +import attr +import marshmallow +import yaml import modmail.config +MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent +ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" + + +def export_default_conf() -> None: + """Export default configuration as both toml and yaml to the preconfigured locations.""" + conf = modmail.config.get_default_config() + dump: dict = modmail.config.ConfigurationSchema().dump(conf) + + # Sort the dictionary configuration. + # This is the only place where the order of the config should matter, when exporting in a specific style + def sort_dict(d: dict) -> dict: + """Takes a dict and sorts it, recursively.""" + sorted_dict = {x[0]: x[1] for x in sorted(d.items(), key=lambda e: e[0])} + + for k, v in d.items(): + if not isinstance(v, dict): + continue + sorted_dict[k] = sort_dict(v) + + return sorted_dict + + dump = sort_dict(dump) + autogen_gen_notice = f"Directly run scripts/{__file__.rsplit('/',1)[-1]!s} to generate." + doc = atoml.document() + doc.add(atoml.comment("This is an autogenerated TOML document.")) + doc.add(atoml.comment(autogen_gen_notice)) + doc.add(atoml.nl()) + + doc.update(dump) + + # toml + + with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml"), "w") as f: + atoml.dump(doc, f) + + # yaml + with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: + f.write("# This is an autogenerated YAML document.\n") + f.write(f"# {autogen_gen_notice}\n") + yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + + +def export_env_conf() -> None: + """ + Exports required configuration variables to .env.template. + + Does NOT export *all* settable variables! + + Export the *required* environment variables to `.env.template`. + Required environment variables are any Config.default.bot variables that default to marshmallow.missing + + Currently supported environment loading only loads to Config.user.bot, so we only need to worry + about those for the time being. Those values are additionally prefixed with `BOT_`. + + This means that in the end our exported variables are all prefixed with MODMAIL_BOT_, + and followed by the uppercase name of each field. + """ + env_prefix = modmail.config.ENV_PREFIX + modmail.config.BOT_ENV_PREFIX + default = modmail.config.get_default_config() + values = defaultdict(str) + fields = attr.fields(default.bot.__class__) + for attribute in fields: + if attribute.default is marshmallow.missing: + values[env_prefix + attribute.name.upper()] = attribute.metadata.get("modmail_export_filler", "") + + with open(ENV_EXPORT_FILE, "w") as f: + for k, v in values.items(): + f.write(k + '="' + v + '"\n') + + +def main() -> None: + """ + Exports the default configuration. + + There's two parts to this export. + First, export the default configuration to the default locations. + + Next, export the *required* configuration variables to the .env.template + + """ + export_default_conf() + + export_env_conf() + + if __name__ == "__main__": print("Exporting configuration to default files. If they exist, overwriting their contents.") - sys.exit(modmail.config.export_default_conf(export_yaml=True)) + sys.exit(main()) From b8d7080dfdc644b39d196cc19f6a7940619a2683 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 02:45:37 -0400 Subject: [PATCH 14/76] tools: add app.json to configuration exports Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 2 +- app.json | 2 +- modmail/config.py | 14 +++++-- modmail/utils/embeds.py | 2 +- modmail/utils/extensions.py | 2 +- .../export_new_config_to_default_config.py | 39 +++++++++++++++---- 6 files changed, 46 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 654e7dae..4f9f925a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: name: Export default configuration language: python entry: poetry run python -m scripts.export_new_config_to_default_config - files: '(\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' additional_dependencies: - atoml~=1.0.3 - pyyaml~=5.4.1 diff --git a/app.json b/app.json index 2a69903a..00c11d9a 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { "env": { "MODMAIL_BOT_TOKEN": { - "description": "Discord bot token. This is from https://discord.com/developers/applications", + "description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", "required": true } }, diff --git a/modmail/config.py b/modmail/config.py index db95793b..46eb2450 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -142,6 +142,7 @@ class Bot: "dump_only": True, "allow_none": False, "modmail_export_filler": "MyBotToken", + "modmail_env_description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", # noqa: E501 }, ) prefix: str = attr.ib( @@ -219,6 +220,7 @@ class Cfg: _CACHED_CONFIG: "Config" = None +_CACHED_DEFAULT: Cfg = None @attr.s(auto_attribs=True, slots=True, kw_only=True) @@ -396,8 +398,8 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: "Not gonna lie, this SHOULD be unreachable...\n" "If you came across this as a consumer, please report this bug to our bug tracker." ) - loaded_config_dict = ConfigurationSchema().load(loaded_config_dict) - return Config(user=loaded_config_dict, schema=ConfigurationSchema) + loaded_config_dict = ConfigurationSchema().load(loaded_config_dict, unknown=marshmallow.EXCLUDE) + return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) def get_config() -> Config: @@ -414,7 +416,11 @@ def get_config() -> Config: def get_default_config() -> Cfg: """Get the default configuration instance of the global Config instance.""" - return get_config().default + global _CACHED_DEFAULT + if _CACHED_DEFAULT is None: + _CACHED_DEFAULT = Cfg() + return _CACHED_DEFAULT -config = get_config() +config = get_config +default = get_default_config diff --git a/modmail/utils/embeds.py b/modmail/utils/embeds.py index 1e945c4c..9db685a8 100644 --- a/modmail/utils/embeds.py +++ b/modmail/utils/embeds.py @@ -6,7 +6,7 @@ from modmail.config import config -DEFAULT_COLOR = config.user.colours.base_embed_color +DEFAULT_COLOR = config().user.colours.base_embed_color original_init = discord.Embed.__init__ diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index 23f10880..b7bf7cae 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -35,7 +35,7 @@ def determine_bot_mode() -> int: """ bot_mode = 0 for mode in BotModes: - if getattr(config.user.dev.mode, unqualify(str(mode)).lower(), True): + if getattr(config().user.dev.mode, unqualify(str(mode)).lower(), True): bot_mode += mode.value return bot_mode diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 26aba0ef..34b22355 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -3,8 +3,10 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ +import json import pathlib import sys +import typing from collections import defaultdict import atoml @@ -17,6 +19,8 @@ MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" +APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" +METADATA_PREFIX = "modmail_" def export_default_conf() -> None: @@ -58,7 +62,7 @@ def sort_dict(d: dict) -> dict: yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) -def export_env_conf() -> None: +def export_env_and_app_json_conf() -> None: """ Exports required configuration variables to .env.template. @@ -75,30 +79,51 @@ def export_env_conf() -> None: """ env_prefix = modmail.config.ENV_PREFIX + modmail.config.BOT_ENV_PREFIX default = modmail.config.get_default_config() - values = defaultdict(str) + req_env_values: typing.Dict[str, attr.Attribute.metadata] = dict() fields = attr.fields(default.bot.__class__) for attribute in fields: if attribute.default is marshmallow.missing: - values[env_prefix + attribute.name.upper()] = attribute.metadata.get("modmail_export_filler", "") + req_env_values[env_prefix + attribute.name.upper()] = defaultdict(str, attribute.metadata) with open(ENV_EXPORT_FILE, "w") as f: - for k, v in values.items(): - f.write(k + '="' + v + '"\n') + for k, v in req_env_values.items(): + f.write(k + '="' + v[METADATA_PREFIX + "export_filler"] + '"\n') + + # the rest of this is designated for the app.json file + with open(APP_JSON_FILE) as f: + try: + app_json: typing.Dict = json.load(f) + except Exception as e: + print( + "Oops! Please ensure the app.json file is valid json! " + "If you've made manual edits, you may want to revert them." + ) + raise e + app_json_env = defaultdict(str) + for env_var, meta in req_env_values.items(): + app_json_env[env_var] = defaultdict( + str, {"description": meta[METADATA_PREFIX + "env_description"], "required": True} + ) + app_json["env"] = app_json_env + with open(APP_JSON_FILE, "w") as f: + json.dump(app_json, f, indent=4) + f.write("\n") def main() -> None: """ Exports the default configuration. - There's two parts to this export. + There's several parts to this export. First, export the default configuration to the default locations. Next, export the *required* configuration variables to the .env.template + In addition, export to app.json when exporting .env.template. """ export_default_conf() - export_env_conf() + export_env_and_app_json_conf() if __name__ == "__main__": From 19ddc22d7ba584c68ab10d9b630616099cce62b5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:00:47 -0400 Subject: [PATCH 15/76] minor: manaually update references to configuration files Signed-off-by: onerandomusername --- docs/contributing.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 3c0bad0b..4f351a13 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -152,14 +152,14 @@ $ poetry install ### Set up modmail config -1. Create a copy of `config-default.yml` named `config.yml` in the the `modmail/` directory. +1. Create a copy of `modmail/default_config.toml` named `modmail_config.toml` in the root of the repository. === "Linux, macOS"
```console - $ cp -v modmail/config-default.toml modmail/config.toml + $ cp -v modmail/default_config.toml modmail_config.toml ```
@@ -169,11 +169,15 @@ $ poetry install
```console - $ xcopy /f modmail/config-default.toml modmail/config.toml + $ xcopy /f modmail/default_config.toml modmail_config.toml ```
+!!! note + If you would like, there is optional support for yaml configuration. Make sure that pyyaml is installed with `poetry install --extras yaml`, + and copy modmail/defaults_config.yaml to modmail_config.yaml. + 2. Set the modmail bot prefix in `bot.prefix`. 3. In case you are a contributor set `dev.mode.plugin_dev` and `dev.mode.develop` to `true`. The `develop` variable enables the developer bot extensions and `plugin_dev` enables plugin-developer friendly bot extensions. 4. Create a text file named `.env` in your project root (that's the base folder of your repository): @@ -182,7 +186,7 @@ $ poetry install !!!note The entire file name is literally `.env` -5. Open the file with any text editor and write the bot token to the files in this format: `TOKEN="my_token"`. +5. Open the file with any text editor and write the bot token to the files in this format: `MODMAIL_BOT_TOKEN="my_token"`. ### Run The Project From 06aad80cabf128ead07bebd64b24df28c50001a3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:12:33 -0400 Subject: [PATCH 16/76] fix: add missing modmail dependencies to additional hook dependencies Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f9f925a..3c591291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,12 @@ repos: files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' additional_dependencies: - atoml~=1.0.3 + - attrs~=21.2.0 + - coloredlogs + - desert + - discord.py + - environs + - marshmallow~=3.13.0 - pyyaml~=5.4.1 - repo: https://github.com/pre-commit/pre-commit-hooks From cc502621f6f88fff50cfe3df6043c8e9a92d830e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:46:40 -0400 Subject: [PATCH 17/76] fix: no configuration files now loads the default config Signed-off-by: onerandomusername --- modmail/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 46eb2450..9596f351 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -394,10 +394,11 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: ) if loaded_config_dict is None: - raise CfgLoadError( - "Not gonna lie, this SHOULD be unreachable...\n" - "If you came across this as a consumer, please report this bug to our bug tracker." - ) + # load default configuration since no overrides were provided. + # that's actually done by default, so we replace the new dictionary + # with the one containing our required environment. + loaded_config_dict = env_cfg + loaded_config_dict = ConfigurationSchema().load(loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From bdc7d42a45399264d55fb3423b3e884573049972 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 03:47:21 -0400 Subject: [PATCH 18/76] fix: create a Config instance when adding it to the bot Signed-off-by: onerandomusername --- modmail/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/bot.py b/modmail/bot.py index 8d30426f..fb89711a 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -37,7 +37,7 @@ class ModmailBot(commands.Bot): logger: ModmailLogger = logging.getLogger(__name__) def __init__(self, **kwargs): - self.config = config + self.config = config() self.start_time: t.Optional[arrow.Arrow] = None # arrow.utcnow() self.http_session: t.Optional[ClientSession] = None From 29de36ee11d51a9232dc7a682f63526facda686c Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 15:15:11 -0400 Subject: [PATCH 19/76] fix: load prefix from modmail_config.toml if not in .env Signed-off-by: onerandomusername --- .env.template | 1 + .pre-commit-config.yaml | 1 + app.json | 4 ++++ modmail/config.py | 47 ++++++++++++++++++++++++----------------- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/.env.template b/.env.template index 66c582eb..521ba21f 100644 --- a/.env.template +++ b/.env.template @@ -1 +1,2 @@ MODMAIL_BOT_TOKEN="MyBotToken" +MODMAIL_BOT_PREFIX="" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c591291..f06fe14f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,7 @@ repos: - id: check-yaml exclude: 'mkdocs.yml' # Exclude all mkdocs.yml as they use tags i.e. `!!!` - id: pretty-format-json + exclude: 'app.json' args: [--indent=4, --autofix] - id: end-of-file-fixer - id: no-commit-to-branch diff --git a/app.json b/app.json index 00c11d9a..d48489cc 100644 --- a/app.json +++ b/app.json @@ -3,6 +3,10 @@ "MODMAIL_BOT_TOKEN": { "description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", "required": true + }, + "MODMAIL_BOT_PREFIX": { + "description": "", + "required": true } }, "name": "Modmail Bot", diff --git a/modmail/config.py b/modmail/config.py index 9596f351..dd714c27 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -60,6 +60,21 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) +def _recursive_dict_update(d1: dict, d2: dict) -> defaultdict: + """ + Recursively update a dictionary with the values from another dictionary. + + Serves to ensure that all keys from both exist. + """ + d1.update(d2) + for k, v in d1.items(): + if isinstance(v, dict) and isinstance(d2.get(k, None), dict): + d1[k] = _recursive_dict_update(v, d2[k]) + elif v is marshmallow.missing and d2.get(k, marshmallow.missing) is not marshmallow.missing: + d1[k] = d2[k] + return defaultdict(lambda: marshmallow.missing, d1) + + class CfgLoadError(Exception): """Exception if the configuration failed to load from a local file.""" @@ -146,11 +161,8 @@ class Bot: }, ) prefix: str = attr.ib( - default="?", - converter=lambda x: "?" if x is None else x, - metadata={ - "allow_none": False, - }, + default=marshmallow.missing, + converter=lambda x: "?" if x is marshmallow.missing else x, ) @@ -266,6 +278,8 @@ def _build_bot_class( kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) if defaults and kw[var.name] is None: kw[var.name] = defaults[var.name] + elif kw[var.name] is None: + kw[var.name] = marshmallow.missing return klass(**kw) @@ -287,7 +301,9 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict["bot"].update(attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX))) + existing_cfg_dict["bot"] = _recursive_dict_update( + attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)), existing_cfg_dict["bot"] + ) return existing_cfg_dict @@ -311,8 +327,8 @@ def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defau with open(path) as f: loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) if existing_cfg_dict is not None: - loaded_cfg.update(existing_cfg_dict) - return existing_cfg_dict + _recursive_dict_update(loaded_cfg, existing_cfg_dict) + return loaded_cfg else: return loaded_cfg except Exception as e: @@ -362,11 +378,7 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: Supported file types are .toml or .yaml """ - # load the env first - if load_env: - env_cfg = _load_env() - else: - env_cfg = None + env_cfg = None if files is None: files = DEFAULT_CONFIG_FILES @@ -393,13 +405,10 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: "the required dependencies are not installed." ) - if loaded_config_dict is None: - # load default configuration since no overrides were provided. - # that's actually done by default, so we replace the new dictionary - # with the one containing our required environment. - loaded_config_dict = env_cfg + if load_env: + loaded_config_dict = _load_env(existing_cfg_dict=loaded_config_dict) - loaded_config_dict = ConfigurationSchema().load(loaded_config_dict, unknown=marshmallow.EXCLUDE) + loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From 2166f8c7844f939bd0cf1d54d4615e8956cfc262 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 16:54:11 -0400 Subject: [PATCH 20/76] fix: ignore extra configuration variables Signed-off-by: onerandomusername --- modmail/config.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/modmail/config.py b/modmail/config.py index dd714c27..e45d86e2 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -302,7 +302,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d existing_cfg_dict = defaultdict(_generate_default_dict) existing_cfg_dict["bot"] = _recursive_dict_update( - attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)), existing_cfg_dict["bot"] + existing_cfg_dict["bot"], attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)) ) return existing_cfg_dict @@ -369,6 +369,29 @@ def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: raise CfgLoadError from e +DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) + + +def _remove_extra_values(klass: type, dit: DictT) -> DictT: + """ + Remove extra values from the provided dict which don't fit into the provided klass recursively. + + klass must be an attr.s class. + """ + fields = attr.fields_dict(klass) + cleared_dict = dit.copy() + for k in dit: + if k not in fields: + del cleared_dict[k] + elif isinstance(cleared_dict[k], dict): + if attr.has((new_klass := fields.get(k, None)).type): + cleared_dict[k] = _remove_extra_values(new_klass.type, cleared_dict[k]) + else: + # delete this dict + del cleared_dict[k] + return cleared_dict + + def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: bool = True) -> Config: """ Loads a configuration from the specified files. @@ -408,6 +431,12 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: if load_env: loaded_config_dict = _load_env(existing_cfg_dict=loaded_config_dict) + # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them + # CONTRARY to the marshmallow.EXCLUDE below. + # They will cause errors. + # Extra configuration values are okay, we aren't trying to be strict here. + loaded_config_dict = _remove_extra_values(Cfg, loaded_config_dict) + loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From be6b698393d5cfc5c8fcb70e9b8e206d7f54b7d9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 19:13:30 -0400 Subject: [PATCH 21/76] minor: use attr helper method instead of reimplementing it checked with a debug command while in a debug session '{x for x in attr.fields(klass)} == attribs' is True. They are exactly the same, so I've switched to using the attr method instead of rolling my own. Signed-off-by: onerandomusername --- modmail/config.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index e45d86e2..2c97014a 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -266,15 +266,10 @@ def _build_bot_class( defaults = defaultdict(lambda: marshmallow.missing) else: defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) - attribs: typing.Set[attr.Attribute] = set() - for a in dir(klass.__attrs_attrs__): - if hasattr(klass.__attrs_attrs__, a): - if isinstance(attribute := getattr(klass.__attrs_attrs__, a), attr.Attribute): - attribs.add(attribute) - # read the env vars from the above + with env.prefixed(ENV_PREFIX): kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing - for var in attribs: + for var in attr.fields(klass): kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) if defaults and kw[var.name] is None: kw[var.name] = defaults[var.name] From 4a367c4c42329af8a4a85947246038886300b9e1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 19:14:42 -0400 Subject: [PATCH 22/76] chore: switch _load_config to pack positional args rather than require an iterable Signed-off-by: onerandomusername --- modmail/config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 2c97014a..97143ed7 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -387,7 +387,7 @@ def _remove_extra_values(klass: type, dit: DictT) -> DictT: return cleared_dict -def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: bool = True) -> Config: +def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: """ Loads a configuration from the specified files. @@ -398,10 +398,8 @@ def _load_config(files: typing.List[typing.Union[os.PathLike]] = None, load_env: """ env_cfg = None - if files is None: + if len(files) == 0: files = DEFAULT_CONFIG_FILES - elif len(files) == 0: - raise CfgLoadError("At least one file to load from must be provided.") loaded_config_dict: dict = None for file in files: From 4e085d72e32db2f41828a7b944c5c59dea54fab9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 20:13:41 -0400 Subject: [PATCH 23/76] fix: rewrite environment variable parser to support all configuration options Signed-off-by: onerandomusername --- modmail/config.py | 64 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 97143ed7..7af74526 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -10,7 +10,7 @@ import desert import discord import discord.ext.commands.converter -import environs +import dotenv import marshmallow import marshmallow.fields import marshmallow.validate @@ -54,6 +54,11 @@ _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] +# load env before we do *anything* +# TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. +# TODO: add the above feature to the other configuration locations too. +dotenv.load_dotenv(_CWD / ".env") + def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" @@ -252,29 +257,57 @@ class Config: default: Cfg = Cfg() +ClassT = typing.TypeVar("ClassT", bound=type) + + # find and build a bot class from our env -def _build_bot_class( - klass: typing.Any, env: environs.Env, class_prefix: str = "", defaults: typing.Dict = None -) -> Bot: +def _build_class( + klass: ClassT, + env: typing.Dict[str, str] = None, + env_prefix: str = None, + *, + dotenv_file: os.PathLike = None, + defaults: typing.Dict = None, +) -> ClassT: """ Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. + Defaults to getting the environment variables with dotenv. + Also can parse from a provided dictionary of environment variables. If defaults is provided, uses a value from there if the environment variable is not set or is None. """ + if env_prefix is None: + env_prefix = ENV_PREFIX + + if env is None: + if dotenv_file is not None: + dotenv.load_dotenv(dotenv_file) + env = os.environ.copy() + # get the attributes of the provided class if defaults is None: defaults = defaultdict(lambda: marshmallow.missing) else: defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) - with env.prefixed(ENV_PREFIX): - kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing - for var in attr.fields(klass): - kw[var.name] = getattr(env, var.type.__name__)(class_prefix + var.name.upper()) - if defaults and kw[var.name] is None: - kw[var.name] = defaults[var.name] - elif kw[var.name] is None: - kw[var.name] = marshmallow.missing + kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing + + for var in attr.fields(klass): + if attr.has(var.type): + # var is an attrs class too + kw[var.name] = _build_class( + var.type, + env=env, + env_prefix=env_prefix + var.name.upper() + "_", + defaults=defaults.get(var.name, None), + ) + else: + kw[var.name] = env.get(env_prefix + var.name.upper(), None) + if kw[var.name] is None: + if defaults: + kw[var.name] = defaults[var.name] + else: + del kw[var.name] return klass(**kw) @@ -290,14 +323,11 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d else: env_file = pathlib.Path(env_file) - env = environs.Env(eager=False, expand_vars=True) - env.read_env(".env", recurse=False) - if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict["bot"] = _recursive_dict_update( - existing_cfg_dict["bot"], attr.asdict(_build_bot_class(Bot, env, BOT_ENV_PREFIX)) + existing_cfg_dict = _recursive_dict_update( + existing_cfg_dict, attr.asdict(_build_class(Cfg, dotenv_file=env_file)) ) return existing_cfg_dict From 449593838a5896fdce782642b5d5c267c8bcf42b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 21:26:30 -0400 Subject: [PATCH 24/76] feat: add ConfigMetadata part 1 Signed-off-by: onerandomusername --- .env.template | 3 +- .pre-commit-config.yaml | 1 + app.json | 6 +-- modmail/config.py | 52 +++++++++++++++++-- .../export_new_config_to_default_config.py | 16 ++++-- 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/.env.template b/.env.template index 521ba21f..d8615894 100644 --- a/.env.template +++ b/.env.template @@ -1,2 +1 @@ -MODMAIL_BOT_TOKEN="MyBotToken" -MODMAIL_BOT_PREFIX="" +MODMAIL_BOT_TOKEN='Bot Token' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f06fe14f..b6136883 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: - discord.py - environs - marshmallow~=3.13.0 + - python-dotenv - pyyaml~=5.4.1 - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/app.json b/app.json index d48489cc..b3b04bc4 100644 --- a/app.json +++ b/app.json @@ -1,11 +1,7 @@ { "env": { "MODMAIL_BOT_TOKEN": { - "description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", - "required": true - }, - "MODMAIL_BOT_PREFIX": { - "description": "", + "description": "Discord bot token. Required to log in to discord.", "required": true } }, diff --git a/modmail/config.py b/modmail/config.py index 7af74526..26d7aa23 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -44,7 +44,7 @@ ] _CWD = pathlib.Path.cwd() - +METADATA_TABLE = "modmail_metadata" ENV_PREFIX = "MODMAIL_" BOT_ENV_PREFIX = "BOT_" USER_CONFIG_FILE_NAME = "modmail_config" @@ -144,6 +144,38 @@ def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Col return _ColourField.ColourConvert().convert(col) +@attr.dataclass(frozen=True, kw_only=True) +class ConfigMetadata: + """ + Cfg metadata. This is intended to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. + + Nearly all of these values are optional, save for the description. + All of them are keyword only. + In addition, all instances of this class are frozen; they are not meant to be changed after creation. + + These values are meant to be used for the configuration UI and exports. + In relation to that, most are optional. However, *all* instances must have a description. + """ + + # what to refer to for the end user + description: str = attr.ib() + canconical_name: str = None + # for those configuration options where the description just won't cut it. + extended_description: str = None + # for the variables to export to the environment, what value should be prefilled + export_environment_prefill: str = None + # comment provided above or beside the default configuration option in exports + export_note: str = None + # this only has an effect if required is not True + # required variables will always be exported, but if this is a commonly changed var, this should be True. + export_to_env_template: bool = False + + @description.validator + def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: + if not isinstance(value, attrib.type): + raise ValueError(f"description must be of {attrib.type}") from None + + @attr.s(auto_attribs=True, slots=True) class Bot: """ @@ -161,12 +193,24 @@ class Bot: "required": True, "dump_only": True, "allow_none": False, - "modmail_export_filler": "MyBotToken", - "modmail_env_description": "Discord bot token. This is obtainable from https://discord.com/developers/applications", # noqa: E501 + METADATA_TABLE: ConfigMetadata( + canconical_name="Bot Token", + description="Discord bot token. Required to log in to discord.", + export_environment_prefill="Bot Token", + extended_description="This is obtainable from https://discord.com/developers/applications", + export_to_env_template=True, + ), }, ) prefix: str = attr.ib( - default=marshmallow.missing, + default="?", + metadata={ + METADATA_TABLE: ConfigMetadata( + canconical_name="Command Prefix", + description="Command prefix.", + export_to_env_template=True, + ) + }, converter=lambda x: "?" if x is marshmallow.missing else x, ) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 34b22355..fc6abedb 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -11,6 +11,7 @@ import atoml import attr +import dotenv import marshmallow import yaml @@ -85,9 +86,12 @@ def export_env_and_app_json_conf() -> None: if attribute.default is marshmallow.missing: req_env_values[env_prefix + attribute.name.upper()] = defaultdict(str, attribute.metadata) - with open(ENV_EXPORT_FILE, "w") as f: - for k, v in req_env_values.items(): - f.write(k + '="' + v[METADATA_PREFIX + "export_filler"] + '"\n') + # dotenv modifies currently existing files, but we want to erase the current file + ENV_EXPORT_FILE.unlink(missing_ok=True) + ENV_EXPORT_FILE.touch() + + for k, v in req_env_values.items(): + dotenv.set_key(ENV_EXPORT_FILE, k, v[modmail.config.METADATA_TABLE].export_environment_prefill) # the rest of this is designated for the app.json file with open(APP_JSON_FILE) as f: @@ -102,7 +106,11 @@ def export_env_and_app_json_conf() -> None: app_json_env = defaultdict(str) for env_var, meta in req_env_values.items(): app_json_env[env_var] = defaultdict( - str, {"description": meta[METADATA_PREFIX + "env_description"], "required": True} + str, + { + "description": meta[modmail.config.METADATA_TABLE].description, + "required": meta.get("required", True), + }, ) app_json["env"] = app_json_env with open(APP_JSON_FILE, "w") as f: From 0085f2aee87e94edd1b2e11c9a5f2db297d1b0b5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 21:39:28 -0400 Subject: [PATCH 25/76] fix recursive dict update overriding set values Signed-off-by: onerandomusername --- modmail/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 26d7aa23..5c002797 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -71,12 +71,17 @@ def _recursive_dict_update(d1: dict, d2: dict) -> defaultdict: Serves to ensure that all keys from both exist. """ - d1.update(d2) for k, v in d1.items(): if isinstance(v, dict) and isinstance(d2.get(k, None), dict): d1[k] = _recursive_dict_update(v, d2[k]) - elif v is marshmallow.missing and d2.get(k, marshmallow.missing) is not marshmallow.missing: + elif (v is marshmallow.missing or v is None) and d2.get( + k, marshmallow.missing + ) is not marshmallow.missing: d1[k] = d2[k] + for k, v in d2.items(): + no = d1.get(k, None) + if no is marshmallow.missing or no is None: + d1[k] = v return defaultdict(lambda: marshmallow.missing, d1) From 9417d3c199c7dce601d82b8d88418a9d5ab05502 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 21:45:48 -0400 Subject: [PATCH 26/76] chore: update export docstring Signed-off-by: onerandomusername --- scripts/export_new_config_to_default_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index fc6abedb..10d9bbb1 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -72,8 +72,9 @@ def export_env_and_app_json_conf() -> None: Export the *required* environment variables to `.env.template`. Required environment variables are any Config.default.bot variables that default to marshmallow.missing - Currently supported environment loading only loads to Config.user.bot, so we only need to worry - about those for the time being. Those values are additionally prefixed with `BOT_`. + TODO: as of right now, all configuration values can be configured with environment variables. + However, this method only exports the MODMAIL_BOT_ *required* varaibles to the template files. + This will be rewritten to support full unload and the new modmail.config.ConfigMetadata class. This means that in the end our exported variables are all prefixed with MODMAIL_BOT_, and followed by the uppercase name of each field. From bb1d956eee649d1dd071ad52d7242cb9ec874405 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 22:48:45 -0400 Subject: [PATCH 27/76] fix: respect priorty of configuration sources Signed-off-by: onerandomusername --- .env.template | 1 + app.json | 5 +++ modmail/config.py | 37 ++++++++++++------- .../export_new_config_to_default_config.py | 8 +++- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.env.template b/.env.template index d8615894..03a02729 100644 --- a/.env.template +++ b/.env.template @@ -1 +1,2 @@ MODMAIL_BOT_TOKEN='Bot Token' +MODMAIL_BOT_PREFIX='?' diff --git a/app.json b/app.json index b3b04bc4..e0d64443 100644 --- a/app.json +++ b/app.json @@ -3,6 +3,11 @@ "MODMAIL_BOT_TOKEN": { "description": "Discord bot token. Required to log in to discord.", "required": true + }, + "MODMAIL_BOT_PREFIX": { + "description": "Command prefix.", + "required": false, + "value": "?" } }, "name": "Modmail Bot", diff --git a/modmail/config.py b/modmail/config.py index 5c002797..d25d7986 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -65,7 +65,7 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -def _recursive_dict_update(d1: dict, d2: dict) -> defaultdict: +def _recursive_dict_update(d1: dict, d2: dict, attr_cls: type = None) -> defaultdict: """ Recursively update a dictionary with the values from another dictionary. @@ -175,6 +175,10 @@ class ConfigMetadata: # required variables will always be exported, but if this is a commonly changed var, this should be True. export_to_env_template: bool = False + # app json is slightly different, so there's additional options for them + app_json_default: str = None + app_json_required: bool = None + @description.validator def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: if not isinstance(value, attrib.type): @@ -208,11 +212,14 @@ class Bot: }, ) prefix: str = attr.ib( - default="?", + default=marshmallow.missing, metadata={ METADATA_TABLE: ConfigMetadata( canconical_name="Command Prefix", description="Command prefix.", + export_environment_prefill="?", + app_json_default="?", + app_json_required=False, export_to_env_template=True, ) }, @@ -261,7 +268,7 @@ class DevCfg: log_level: int = attr.ib(default=logging.INFO) @log_level.validator - def _log_level_validator(self, _: attr.Attribute, value: int) -> None: + def _log_level_validator(self, a: attr.Attribute, value: int) -> None: """Validate that log_level is within 0 to 50.""" if value not in range(0, 50 + 1): raise ValueError("log_level must be an integer within 0 to 50, inclusive.") @@ -335,11 +342,11 @@ def _build_class( # get the attributes of the provided class if defaults is None: - defaults = defaultdict(lambda: marshmallow.missing) + defaults = defaultdict(lambda: None) else: - defaults = defaultdict(lambda: marshmallow.missing, defaults.copy()) + defaults = defaultdict(lambda: None, defaults.copy()) - kw = defaultdict(lambda: marshmallow.missing) # any missing required vars provide as missing + kw = defaultdict(lambda: None) # any missing required vars provide as missing for var in attr.fields(klass): if attr.has(var.type): @@ -353,8 +360,12 @@ def _build_class( else: kw[var.name] = env.get(env_prefix + var.name.upper(), None) if kw[var.name] is None: - if defaults: + if defaults is not None and ( + (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing + ): kw[var.name] = defaults[var.name] + elif var.default is not attr.NOTHING: # check for var default + kw[var.name] = var.default else: del kw[var.name] @@ -375,9 +386,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict = _recursive_dict_update( - existing_cfg_dict, attr.asdict(_build_class(Cfg, dotenv_file=env_file)) - ) + existing_cfg_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) return existing_cfg_dict @@ -401,7 +410,7 @@ def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defau with open(path) as f: loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) if existing_cfg_dict is not None: - _recursive_dict_update(loaded_cfg, existing_cfg_dict) + loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) return loaded_cfg else: return loaded_cfg @@ -433,10 +442,10 @@ def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: try: with open(path, "r") as f: - loaded_cfg = dict(yaml.load(f.read(), Loader=yaml.SafeLoader)) + loaded_cfg = defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) if existing_cfg_dict is not None: - loaded_cfg.update(existing_cfg_dict) - return existing_cfg_dict + loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) + return loaded_cfg else: return loaded_cfg except Exception as e: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 10d9bbb1..584563db 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -106,13 +106,17 @@ def export_env_and_app_json_conf() -> None: raise e app_json_env = defaultdict(str) for env_var, meta in req_env_values.items(): - app_json_env[env_var] = defaultdict( + options = defaultdict( str, { "description": meta[modmail.config.METADATA_TABLE].description, - "required": meta.get("required", True), + "required": meta[modmail.config.METADATA_TABLE].app_json_required + or meta.get("required", False), }, ) + if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: + options["value"] = value + app_json_env[env_var] = options app_json["env"] = app_json_env with open(APP_JSON_FILE, "w") as f: json.dump(app_json, f, indent=4) From d5f1290960ac33e9093bca6cda96e56ce13674f9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 20 Sep 2021 23:00:17 -0400 Subject: [PATCH 28/76] fix pre-commit maybe Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6136883..ed670ebc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: language: python entry: poetry run python -m scripts.export_new_config_to_default_config files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + require_serial: true additional_dependencies: - atoml~=1.0.3 - attrs~=21.2.0 From f3c3326b8260cea57cd12d405a28d98e63d59c56 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 03:41:52 -0400 Subject: [PATCH 29/76] nit: rename config class Bot to BotCfg to prevent future confusion Signed-off-by: onerandomusername --- modmail/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index d25d7986..5f4fddfb 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -31,7 +31,7 @@ "Config", "config", "ConfigurationSchema", - "Bot", + "BotCfg", "BotModeCfg", "Cfg", "Colours", @@ -186,7 +186,7 @@ def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> No @attr.s(auto_attribs=True, slots=True) -class Bot: +class BotCfg: """ Values that are configuration for the bot itself. @@ -283,7 +283,7 @@ class Cfg: we can get a clean default variable if we don't pass anything. """ - bot: Bot = Bot() + bot: BotCfg = BotCfg() colours: Colours = Colours() dev: DevCfg = DevCfg() From 6e14be0b1740dcd33c28a0322476cf93d9ef772c Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 04:14:13 -0400 Subject: [PATCH 30/76] fix: add ConfigMetadata part 2 Signed-off-by: onerandomusername --- modmail/config.py | 64 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 5f4fddfb..e8e33578 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -149,7 +149,7 @@ def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Col return _ColourField.ColourConvert().convert(col) -@attr.dataclass(frozen=True, kw_only=True) +@attr.frozen(kw_only=True) class ConfigMetadata: """ Cfg metadata. This is intended to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. @@ -179,6 +179,10 @@ class ConfigMetadata: app_json_default: str = None app_json_required: bool = None + # hidden, eg log_level + # hidden values mean they do not show up in the bot configuration menu + hidden: bool = False + @description.validator def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: if not isinstance(value, attrib.type): @@ -241,10 +245,32 @@ class BotModeCfg: marshmallow.fields.Constant(True), default=True, converter=lambda _: True, - metadata={"dump_default": True, "dump_only": True}, + metadata={ + "dump_default": True, + "dump_only": True, + METADATA_TABLE: ConfigMetadata( + description="Production Mode. This is not changeable.", + ), + }, + ) + develop: bool = attr.ib( + default=False, + metadata={ + "allow_none": False, + METADATA_TABLE: ConfigMetadata( + description="Bot developer mode. Enables additional developer specific features.", + ), + }, + ) + plugin_dev: bool = attr.ib( + default=False, + metadata={ + "allow_none": False, + METADATA_TABLE: ConfigMetadata( + description="Plugin developer mode. Enables additional plugin developer specific features.", + ), + }, ) - develop: bool = attr.ib(default=False, metadata={"allow_none": False}) - plugin_dev: bool = attr.ib(default=False, metadata={"allow_none": False}) @attr.s(auto_attribs=True, slots=True) @@ -256,7 +282,14 @@ class Colours: """ base_embed_color: discord.Colour = desert.ib( - _ColourField(), default="0x7289DA", converter=convert_to_color + _ColourField(), + default="0x7289DA", + converter=convert_to_color, + metadata={ + METADATA_TABLE: ConfigMetadata( + description="Default embed colour for all embeds without a designated colour.", + ) + }, ) @@ -265,7 +298,15 @@ class DevCfg: """Developer configuration. These values should not be changed unless you know what you're doing.""" mode: BotModeCfg = BotModeCfg() - log_level: int = attr.ib(default=logging.INFO) + log_level: int = attr.ib( + default=logging.INFO, + metadata={ + METADATA_TABLE: ConfigMetadata( + description="Logging level.", + hidden=True, + ) + }, + ) @log_level.validator def _log_level_validator(self, a: attr.Attribute, value: int) -> None: @@ -285,7 +326,16 @@ class Cfg: bot: BotCfg = BotCfg() colours: Colours = Colours() - dev: DevCfg = DevCfg() + dev: DevCfg = attr.ib( + default=DevCfg(), + metadata={ + METADATA_TABLE: ConfigMetadata( + description="Developer configuration. " + "Only change these values if you know what you're doing.", + hidden=True, + ) + }, + ) # build configuration From c05e0bb3715e7d321e059d2a4f7e2aad942a8256 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 04:34:41 -0400 Subject: [PATCH 31/76] chore: remove unused dict manipulation method removed the now defunct _recursive_dict_update method Signed-off-by: onerandomusername --- modmail/config.py | 44 ++++++-------------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index e8e33578..3d4cd900 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -65,26 +65,6 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -def _recursive_dict_update(d1: dict, d2: dict, attr_cls: type = None) -> defaultdict: - """ - Recursively update a dictionary with the values from another dictionary. - - Serves to ensure that all keys from both exist. - """ - for k, v in d1.items(): - if isinstance(v, dict) and isinstance(d2.get(k, None), dict): - d1[k] = _recursive_dict_update(v, d2[k]) - elif (v is marshmallow.missing or v is None) and d2.get( - k, marshmallow.missing - ) is not marshmallow.missing: - d1[k] = d2[k] - for k, v in d2.items(): - no = d1.get(k, None) - if no is marshmallow.missing or no is None: - d1[k] = v - return defaultdict(lambda: marshmallow.missing, d1) - - class CfgLoadError(Exception): """Exception if the configuration failed to load from a local file.""" @@ -441,7 +421,7 @@ def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> d return existing_cfg_dict -def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defaultdict: +def load_toml(path: os.PathLike = None) -> defaultdict: """ Load a configuration dictionary from the specified toml file. @@ -458,17 +438,12 @@ def load_toml(path: os.PathLike = None, existing_cfg_dict: dict = None) -> defau try: with open(path) as f: - loaded_cfg = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) - if existing_cfg_dict is not None: - loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) - return loaded_cfg - else: - return loaded_cfg + return defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) except Exception as e: raise CfgLoadError from e -def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: +def load_yaml(path: os.PathLike) -> dict: """ Load a configuration dictionary from the specified yaml file. @@ -492,12 +467,7 @@ def load_yaml(path: os.PathLike, existing_cfg_dict: dict = None) -> dict: try: with open(path, "r") as f: - loaded_cfg = defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) - if existing_cfg_dict is not None: - loaded_cfg = _recursive_dict_update(loaded_cfg, existing_cfg_dict) - return loaded_cfg - else: - return loaded_cfg + return defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) except Exception as e: raise CfgLoadError from e @@ -534,8 +504,6 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: Supported file types are .toml or .yaml """ - env_cfg = None - if len(files) == 0: files = DEFAULT_CONFIG_FILES @@ -548,10 +516,10 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: continue if file.suffix == ".toml" and atoml is not None: - loaded_config_dict = load_toml(file, existing_cfg_dict=env_cfg) + loaded_config_dict = load_toml(file) break elif file.suffix == ".yaml" and yaml is not None: - loaded_config_dict = load_yaml(file, existing_cfg_dict=env_cfg) + loaded_config_dict = load_yaml(file) break else: raise Exception( From 444aca17898952e69557ac8b68a16d887ba627dd Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 18:12:13 -0400 Subject: [PATCH 32/76] chore: implement new metadata options on cfg default export Signed-off-by: onerandomusername --- modmail/config.py | 9 +- .../export_new_config_to_default_config.py | 102 ++++++++++++------ 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 3d4cd900..4e6ae89d 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -23,10 +23,9 @@ __all__ = [ "AUTO_GEN_FILE_NAME", - "DEFAULT_CONFIG_FILES", "ENV_PREFIX", - "BOT_ENV_PREFIX", "USER_CONFIG_FILE_NAME", + "USER_CONFIG_FILES", "CfgLoadError", "Config", "config", @@ -46,10 +45,9 @@ _CWD = pathlib.Path.cwd() METADATA_TABLE = "modmail_metadata" ENV_PREFIX = "MODMAIL_" -BOT_ENV_PREFIX = "BOT_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" -DEFAULT_CONFIG_FILES = [ +USER_CONFIG_FILES = [ _CWD / (USER_CONFIG_FILE_NAME + ".yaml"), _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] @@ -156,6 +154,7 @@ class ConfigMetadata: export_to_env_template: bool = False # app json is slightly different, so there's additional options for them + export_to_app_json: bool = None app_json_default: str = None app_json_required: bool = None @@ -505,7 +504,7 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: Supported file types are .toml or .yaml """ if len(files) == 0: - files = DEFAULT_CONFIG_FILES + files = USER_CONFIG_FILES loaded_config_dict: dict = None for file in files: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 584563db..d3098400 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -12,7 +12,6 @@ import atoml import attr import dotenv -import marshmallow import yaml import modmail.config @@ -21,13 +20,21 @@ MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" -METADATA_PREFIX = "modmail_" + +METADATA_TABLE = modmail.config.METADATA_TABLE + + +class MetadataDict(typing.TypedDict): + """Typed metadata. This has a possible risk given that the modmail_metadata variable is defined.""" + + modmail_metadata: modmail.config.ConfigMetadata + required: bool def export_default_conf() -> None: """Export default configuration as both toml and yaml to the preconfigured locations.""" - conf = modmail.config.get_default_config() - dump: dict = modmail.config.ConfigurationSchema().dump(conf) + default = modmail.config.get_default_config() + dump: dict = modmail.config.ConfigurationSchema().dump(default) # Sort the dictionary configuration. # This is the only place where the order of the config should matter, when exporting in a specific style @@ -70,31 +77,42 @@ def export_env_and_app_json_conf() -> None: Does NOT export *all* settable variables! Export the *required* environment variables to `.env.template`. - Required environment variables are any Config.default.bot variables that default to marshmallow.missing - - TODO: as of right now, all configuration values can be configured with environment variables. - However, this method only exports the MODMAIL_BOT_ *required* varaibles to the template files. - This will be rewritten to support full unload and the new modmail.config.ConfigMetadata class. - - This means that in the end our exported variables are all prefixed with MODMAIL_BOT_, - and followed by the uppercase name of each field. + Required environment variables are any Config.default variables that default to marshmallow.missing + These can also be configured by using the ConfigMetadata options. """ - env_prefix = modmail.config.ENV_PREFIX + modmail.config.BOT_ENV_PREFIX default = modmail.config.get_default_config() - req_env_values: typing.Dict[str, attr.Attribute.metadata] = dict() - fields = attr.fields(default.bot.__class__) - for attribute in fields: - if attribute.default is marshmallow.missing: - req_env_values[env_prefix + attribute.name.upper()] = defaultdict(str, attribute.metadata) + + # find all environment variables to report + def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, MetadataDict]: + + if env_prefix is None: + env_prefix = modmail.config.ENV_PREFIX + + # exact name, default value + export: typing.Dict[str, MetadataDict] = dict() # any missing required vars provide as missing + + for var in attr.fields(klass): + if attr.has(var.type): + # var is an attrs class too, run this on it. + export.update( + get_env_vars( + var.type, + env_prefix=env_prefix + var.name.upper() + "_", + ) + ) + else: + meta: MetadataDict = var.metadata + # put all values in the dict, we'll iterate through them later. + export[env_prefix + var.name.upper()] = meta + + return export # dotenv modifies currently existing files, but we want to erase the current file ENV_EXPORT_FILE.unlink(missing_ok=True) ENV_EXPORT_FILE.touch() - for k, v in req_env_values.items(): - dotenv.set_key(ENV_EXPORT_FILE, k, v[modmail.config.METADATA_TABLE].export_environment_prefill) + exported = get_env_vars(default.__class__) - # the rest of this is designated for the app.json file with open(APP_JSON_FILE) as f: try: app_json: typing.Dict = json.load(f) @@ -104,19 +122,35 @@ def export_env_and_app_json_conf() -> None: "If you've made manual edits, you may want to revert them." ) raise e - app_json_env = defaultdict(str) - for env_var, meta in req_env_values.items(): - options = defaultdict( - str, - { - "description": meta[modmail.config.METADATA_TABLE].description, - "required": meta[modmail.config.METADATA_TABLE].app_json_required - or meta.get("required", False), - }, - ) - if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: - options["value"] = value - app_json_env[env_var] = options + + app_json_env = dict() + + for key, meta in exported.items(): + if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): + + dotenv.set_key( + ENV_EXPORT_FILE, + key, + meta[METADATA_TABLE].export_environment_prefill or meta["default"], + ) + + if ( + meta[METADATA_TABLE].export_to_app_json + or meta[METADATA_TABLE].export_to_env_template + or meta.get("required", False) + ): + + options = defaultdict( + str, + { + "description": meta[METADATA_TABLE].description, + "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), + }, + ) + if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: + options["value"] = value + app_json_env[key] = options + app_json["env"] = app_json_env with open(APP_JSON_FILE, "w") as f: json.dump(app_json, f, indent=4) From 014e9aac39b07cd4c9617c5d4141031f7672f442 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 22:15:25 -0400 Subject: [PATCH 33/76] tests: add config loading tests test if the cofngiruation properly reads from provided files Signed-off-by: onerandomusername --- modmail/config.py | 13 +-- pyproject.toml | 7 ++ tests/modmail/test_config.py | 150 +++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 tests/modmail/test_config.py diff --git a/modmail/config.py b/modmail/config.py index 4e6ae89d..0d69e9fc 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -18,7 +18,7 @@ try: import yaml -except ImportError: +except ImportError: # pragma: nocover yaml = None __all__ = [ @@ -38,6 +38,7 @@ "convert_to_color", "get_config", "get_default_config", + "load_env", "load_toml", "load_yaml", ] @@ -164,7 +165,7 @@ class ConfigMetadata: @description.validator def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> None: - if not isinstance(value, attrib.type): + if not isinstance(value, attrib.type): # pragma: no branch raise ValueError(f"description must be of {attrib.type}") from None @@ -401,7 +402,7 @@ def _build_class( return klass(**kw) -def _load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: +def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: """ Load a configuration dictionary from the specified env file and environment variables. @@ -494,7 +495,7 @@ def _remove_extra_values(klass: type, dit: DictT) -> DictT: return cleared_dict -def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: +def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: """ Loads a configuration from the specified files. @@ -526,8 +527,8 @@ def _load_config(*files: os.PathLike, load_env: bool = True) -> Config: "the required dependencies are not installed." ) - if load_env: - loaded_config_dict = _load_env(existing_cfg_dict=loaded_config_dict) + if should_load_env: + loaded_config_dict = load_env(existing_cfg_dict=loaded_config_dict) # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. diff --git a/pyproject.toml b/pyproject.toml index 96395a3a..2a48bdec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,13 @@ omit = ["modmail/plugins/**.*"] addopts = "--cov --cov-report=" minversion = "6.0" testpaths = ["tests"] +filterwarnings = [ + "default", + 'ignore::DeprecationWarning:marshmallow.fields:173', + 'ignore::DeprecationWarning:marshmallow.fields:438', + 'ignore::DeprecationWarning:marshmallow.fields:456', +] + [tool.black] line-length = 110 diff --git a/tests/modmail/test_config.py b/tests/modmail/test_config.py new file mode 100644 index 00000000..2548c450 --- /dev/null +++ b/tests/modmail/test_config.py @@ -0,0 +1,150 @@ +import inspect +import os +import pathlib +import textwrap +import typing + +import atoml +import attr +import desert +import discord +import discord.ext.commands.converter +import dotenv +import marshmallow.utils +import pytest + +from modmail import config + + +def test_config_is_cached(): + """Test configuration is cached, helping keep only one version of the configuration in existance.""" + for _ in range(2): + assert config.config() == config._CACHED_CONFIG + + +def test_default_config_is_cached(): + """Test default configuration is cached, helping keep only one version of the config in existance.""" + for _ in range(2): + assert config.default() == config._CACHED_DEFAULT + + +class TestConfigLoaders: + """Test configuration loaders properly read and decode their files.""" + + def test_load_env(self, tmp_path: pathlib.Path): + """ + Ensure an environment variable properly gets loaded. + + This writes a custom .env file and loads them with dotenv. + """ + token = "NjQzOTQ1MjY0ODY4MDk4MDQ5.342b.4inDLBILY69LOLfyi6jk420dpyjoEVsCoModM" # noqa: S105 + prefix = "oop" + dev_mode = True + env = textwrap.dedent( + f""" + MODMAIL_BOT_TOKEN="{token}" + MODMAIL_BOT_PREFIX="{prefix}" + MODMAIL_DEV_MODE_DEVELOP={dev_mode} + """ + ) + test_env = tmp_path / ".env" + with open(test_env, "w") as f: + f.write(env + "\n") + + # we have to run this here since we may have the environment vars in our local env + # but we want to ensure that they are of the above env for the test + dotenv.load_dotenv(test_env, override=True) + cfg_dict = config.load_env(test_env) + + assert token == cfg_dict["bot"]["token"] + assert prefix == cfg_dict["bot"]["prefix"] + assert dev_mode == bool(cfg_dict["dev"]["mode"]["develop"]) + + def test_load_toml(self, tmp_path: pathlib.Path): + """ + Ensure a toml file is loaded is properly loaded. + + This writes a temporary file to the tempfolder and then parses it, deleting it when done. + """ + # toml is a little bit different, so we have to convert our bools to strings + # and then make them lowercase + prefix = "toml_ftw" + log_level = 40 + develop = True + plugin_dev = True + toml = textwrap.dedent( + f""" + [bot] + prefix = "{prefix}" + + [dev] + log_level = {log_level} + + [dev.mode] + develop = {str(develop).lower()} + plugin_dev = {str(plugin_dev).lower()} + """ + ) + test_toml = tmp_path / "test.toml" + with open(test_toml, "w") as f: + f.write(toml + "\n") + + cfg_dict = config.load_toml(test_toml) + + assert prefix == cfg_dict["bot"]["prefix"] + assert log_level == cfg_dict["dev"]["log_level"] + assert develop == bool(cfg_dict["dev"]["mode"]["develop"]) + assert plugin_dev == bool(cfg_dict["dev"]["mode"]["plugin_dev"]) + + def test_load_yaml(self, tmp_path: pathlib.Path): + """ + Ensure a yaml file is loaded is properly loaded. + + This writes a temporary file to the tempfolder and then parses it, deleting it when done. + """ + # this test requires the yaml library to be installed. + _ = pytest.importorskip("yaml", reason="Yaml is not installed, unable to test yaml loading.") + prefix = "toml_ftw" + log_level = 40 + develop = True + plugin_dev = True + yaml = textwrap.dedent( + f""" + bot: + prefix: '{prefix}' + dev: + log_level: {log_level} + mode: + develop: {str(develop).lower()} + plugin_dev: {str(plugin_dev).lower()} + + """ + ) + test_yaml = tmp_path / "test.yaml" + with open(test_yaml, "w") as f: + f.write(yaml + "\n") + + cfg_dict = config.load_yaml(test_yaml) + + assert prefix == cfg_dict["bot"]["prefix"] + assert log_level == cfg_dict["dev"]["log_level"] + assert develop == bool(cfg_dict["dev"]["mode"]["develop"]) + assert plugin_dev == bool(cfg_dict["dev"]["mode"]["plugin_dev"]) + + +def test_colour_conversion(): + """ + Test the discord.py converter takes all supported colours. + + Regression test. + """ + ... + + +def test_metadata_valid(): + """ + Checks that the metadata for every field is valid. + + This is more of a sanity check than anything. + """ + ... From 2a2b9dcc2131dbac0a6a525dca8c202db8c5d703 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 00:24:07 -0400 Subject: [PATCH 34/76] feat: add configuration manager cog Signed-off-by: onerandomusername --- modmail/config.py | 3 +- modmail/extensions/configuration_manager.py | 123 ++++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 modmail/extensions/configuration_manager.py diff --git a/modmail/config.py b/modmail/config.py index 0d69e9fc..78fbaf12 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -196,7 +196,7 @@ class BotCfg: }, ) prefix: str = attr.ib( - default=marshmallow.missing, + default="?", metadata={ METADATA_TABLE: ConfigMetadata( canconical_name="Command Prefix", @@ -207,7 +207,6 @@ class BotCfg: export_to_env_template=True, ) }, - converter=lambda x: "?" if x is marshmallow.missing else x, ) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py new file mode 100644 index 00000000..bed79aec --- /dev/null +++ b/modmail/extensions/configuration_manager.py @@ -0,0 +1,123 @@ +import logging +import string +import typing + +import attr +import marshmallow +from discord.ext import commands +from discord.ext.commands import Context + +from modmail import config +from modmail.bot import ModmailBot +from modmail.log import ModmailLogger +from modmail.utils.cogs import ExtMetadata, ModmailCog +from modmail.utils.pagination import ButtonPaginator + + +EXT_METADATA = ExtMetadata() + +logger: ModmailLogger = logging.getLogger(__name__) + + +@attr.mutable +class ConfOptions: + """Configuration attribute class.""" + + default: str + name: str + description: str + canconical_name: str + extended_description: str + hidden: bool + + _type: type + + nested: str = None + + @classmethod + def from_field(cls, field: attr.Attribute, nested: str = None): + """Create a ConfOptions from a attr.Attribute.""" + kw = {} + kw["default"] = field.default if field.default is not marshmallow.missing else None + kw["name"] = field.name + kw["type"] = field.type + + meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] + kw["description"] = meta.description + kw["canconical_name"] = meta.canconical_name + kw["extended_description"] = meta.extended_description + kw["hidden"] = meta.hidden + + if nested is not None: + kw["nested"] = nested + + return cls(**kw) + + +def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing.Dict[str, ConfOptions]: + """Get a dict of ConfOptions for a designated configuration field recursively.""" + options = dict() + for field in attr.fields(klass): + # make conf option list + if attr.has(field.type): + options.update(get_all_conf_options(field.type, prefix=field.name + ".")) + else: + try: + conf_opt = ConfOptions.from_field(field, nested=prefix) + except KeyError as e: + if field.type == type: + pass + elif config.METADATA_TABLE in e.args[0]: + logger.warn( + f"Issue with field '{field.name}', does not have a {config.METADATA_TABLE} key." + ) + else: + logger.error(f"A key error occured with {field.name}.", exc_info=True) + else: + options[prefix + field.name] = conf_opt + + return options + + +class ConfigurationManager(ModmailCog, name="Configuration Manager"): + """Manage the bot configuration.""" + + config_fields: typing.Dict[str, ConfOptions] + + def __init__(self, bot: ModmailBot): + self.bot = bot + + self.config_fields = get_all_conf_options(config.default().__class__) + + @commands.group(name="config", aliases=("cfg", "conf"), invoke_without_command=True) + async def config_group(self, ctx: Context) -> None: + """Manage the bot configuration.""" + if ctx.invoked_subcommand is None: + await ctx.send_help(ctx.command) + + @config_group.command(name="list") + async def list_config(self, ctx: Context) -> None: + """List the valid configuration options.""" + options = [] + for opt in self.config_fields.values(): + if opt.hidden: + continue + options.append( + "\n".join( + [ + f"**{string.capwords(opt.canconical_name or opt.name)}**", + f"Default: `{opt.default}`" + if opt.default is not None + else "Required. There is no default value for this option.", + f"{opt.description}", + f" {opt.extended_description}" if opt.extended_description else "", + ] + ) + ) + + await ButtonPaginator.paginate(options, ctx.message) + + +def setup(bot: ModmailBot) -> None: + """Load the ConfigurationManager cog.""" + bot.add_cog(ConfigurationManager(bot)) From 9eeee2e2f0e7db21da68b0a7af0fb9aa9cc4abd0 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 08:24:31 -0400 Subject: [PATCH 35/76] migrate some existing values to new configuration Signed-off-by: onerandomusername --- modmail/config.py | 43 +++++++++++++++++++++++++ modmail/default_config.toml | 4 +++ modmail/default_config.yaml | 3 ++ modmail/extensions/extension_manager.py | 25 +++++++++----- modmail/utils/embeds.py | 4 +-- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 78fbaf12..52c6b572 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -9,6 +9,7 @@ import attr import desert import discord +import discord.ext.commands import discord.ext.commands.converter import dotenv import marshmallow @@ -159,6 +160,11 @@ class ConfigMetadata: app_json_default: str = None app_json_required: bool = None + # I have no plans to add async to this file, that would make it overly complex. + # as a solution, I'm implementing a field which can provide a rich converter object, + # in the style that discord.py uses. This will be called like discord py calls. + discord_converter: discord.ext.commands.converter.Converter = attr.ib(default=None) + # hidden, eg log_level # hidden values mean they do not show up in the bot configuration menu hidden: bool = False @@ -168,6 +174,13 @@ def _validate_description(self, attrib: attr.Attribute, value: typing.Any) -> No if not isinstance(value, attrib.type): # pragma: no branch raise ValueError(f"description must be of {attrib.type}") from None + @discord_converter.validator + def _validate_discord_converter(self, attrib: attr.Attribute, value: typing.Any) -> None: + if value is None: + return + if not hasattr(value, "convert"): # pragma: no branch + raise AttributeError("Converters must have a method named convert.") + @attr.s(auto_attribs=True, slots=True) class BotCfg: @@ -294,6 +307,35 @@ def _log_level_validator(self, a: attr.Attribute, value: int) -> None: raise ValueError("log_level must be an integer within 0 to 50, inclusive.") +@attr.mutable(slots=True) +class EmojiCfg: + """ + Emojis used across the entire bot. + + This was a pain to implement. + """ + + success: typing.Any = attr.ib( + default=":thumbsup:", + metadata={ + METADATA_TABLE: ConfigMetadata( + description="This is used in most cases when the bot does a successful action.", + discord_converter=discord.ext.commands.converter.EmojiConverter, + ) + }, + ) + + failure: typing.Any = attr.ib( + default=":x:", + metadata={ + METADATA_TABLE: ConfigMetadata( + description="This is used in most cases when the bot fails an action.", + discord_converter=discord.ext.commands.converter.EmojiConverter, + ) + }, + ) + + @attr.s(auto_attribs=True, slots=True) class Cfg: """ @@ -315,6 +357,7 @@ class Cfg: ) }, ) + emojis: EmojiCfg = EmojiCfg() # build configuration diff --git a/modmail/default_config.toml b/modmail/default_config.toml index de41e516..a3202156 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -15,3 +15,7 @@ log_level = 20 develop = false plugin_dev = false production = true + +[emojis] +failure = ":x:" +success = ":thumbsup:" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index 7ada283b..de7c70cb 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -10,3 +10,6 @@ dev: develop: false plugin_dev: false production: true +emojis: + failure: ':x:' + success: ':thumbsup:' diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index dee0f01b..19554a78 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -11,6 +11,7 @@ from discord.ext import commands from discord.ext.commands import Context +import modmail.config from modmail.bot import ModmailBot from modmail.log import ModmailLogger from modmail.utils.cogs import BotModes, ExtMetadata, ModmailCog @@ -26,6 +27,9 @@ EXT_METADATA = ExtMetadata(load_if_mode=BotModes.DEVELOP, no_unload=True) +Emojis = modmail.config.config().user.emojis + + class Action(Enum): """Represents an action to perform on an extension.""" @@ -66,12 +70,12 @@ async def convert(self, _: Context, argument: str) -> str: matches.append(ext) if not matches: - raise commands.BadArgument(f":x: Could not find the {self.type} `{argument}`.") + raise commands.BadArgument(f"{Emojis.failure} Could not find the {self.type} `{argument}`.") if len(matches) > 1: names = "\n".join(sorted(matches)) raise commands.BadArgument( - f":x: `{argument}` is an ambiguous {self.type} name. " + f"{Emojis.failure} `{argument}` is an ambiguous {self.type} name. " f"Please use one of the following fully-qualified names.```\n{names}```" ) @@ -134,7 +138,9 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) if blacklisted: bl_msg = "\n".join(blacklisted) - await ctx.send(f":x: The following {self.type}(s) may not be unloaded:```\n{bl_msg}```") + await ctx.send( + f"{Emojis.failure} The following {self.type}(s) may not be unloaded:```\n{bl_msg}```" + ) return if "*" in extensions: @@ -253,7 +259,7 @@ def batch_manage(self, action: Action, *extensions: str) -> str: if error: failures[extension] = error - emoji = ":x:" if failures else ":thumbsup:" + emoji = Emojis.failure if failures else Emojis.success msg = f"{emoji} {len(extensions) - len(failures)} / {len(extensions)} {self.type}s {verb}ed." if failures: @@ -274,9 +280,12 @@ def manage(self, action: Action, ext: str) -> t.Tuple[str, t.Optional[str]]: except (commands.ExtensionAlreadyLoaded, commands.ExtensionNotLoaded): if action is Action.RELOAD: # When reloading, have a special error. - msg = f":x: {self.type.capitalize()} `{ext}` is not loaded, so it was not {verb}ed." + msg = ( + f"{Emojis.failure} {self.type.capitalize()} " + f"`{ext}` is not loaded, so it was not {verb}ed." + ) else: - msg = f":x: {self.type.capitalize()} `{ext}` is already {verb}ed." + msg = f"{Emojis.failure} {self.type.capitalize()} `{ext}` is already {verb}ed." except Exception as e: if hasattr(e, "original"): # If original exception is present, then utilize it @@ -285,9 +294,9 @@ def manage(self, action: Action, ext: str) -> t.Tuple[str, t.Optional[str]]: log.exception(f"{self.type.capitalize()} '{ext}' failed to {verb}.") error_msg = f"{e.__class__.__name__}: {e}" - msg = f":x: Failed to {verb} {self.type} `{ext}`:\n```\n{error_msg}```" + msg = f"{Emojis.failure} Failed to {verb} {self.type} `{ext}`:\n```\n{error_msg}```" else: - msg = f":thumbsup: {self.type.capitalize()} successfully {verb}ed: `{ext}`." + msg = f"{Emojis.success} {self.type.capitalize()} successfully {verb}ed: `{ext}`." log.debug(error_msg or msg) return msg, error_msg diff --git a/modmail/utils/embeds.py b/modmail/utils/embeds.py index 9db685a8..70e7b24d 100644 --- a/modmail/utils/embeds.py +++ b/modmail/utils/embeds.py @@ -6,8 +6,6 @@ from modmail.config import config -DEFAULT_COLOR = config().user.colours.base_embed_color - original_init = discord.Embed.__init__ @@ -42,7 +40,7 @@ def __init__(self: discord.Embed, description: str = None, **kwargs): # noqa: N description=description or content or EmptyEmbed, type=kwargs.pop("type", "rich"), url=kwargs.pop("url", EmptyEmbed), - colour=kwargs.pop("color", kwargs.pop("colour", DEFAULT_COLOR)), + colour=kwargs.pop("color", kwargs.pop("colour", config().user.colours.base_embed_color)), timestamp=kwargs.pop("timestamp", EmptyEmbed), ) From d6cbf869231c25eacd975fb9de6a608150ecd357 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 08:25:45 -0400 Subject: [PATCH 36/76] minor: hide frozen values which can't be changed from the configuration Signed-off-by: onerandomusername --- modmail/extensions/configuration_manager.py | 24 ++++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index bed79aec..517c06e3 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -3,6 +3,7 @@ import typing import attr +import attr._make import marshmallow from discord.ext import commands from discord.ext.commands import Context @@ -33,15 +34,18 @@ class ConfOptions: _type: type nested: str = None + frozen: bool = False @classmethod - def from_field(cls, field: attr.Attribute, nested: str = None): + def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str = None): """Create a ConfOptions from a attr.Attribute.""" kw = {} kw["default"] = field.default if field.default is not marshmallow.missing else None kw["name"] = field.name kw["type"] = field.type + kw["frozen"] = field.on_setattr is attr.setters.frozen or frozen + meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] kw["description"] = meta.description kw["canconical_name"] = meta.canconical_name @@ -62,8 +66,9 @@ def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing. if attr.has(field.type): options.update(get_all_conf_options(field.type, prefix=field.name + ".")) else: + is_frozen = klass.__setattr__ is attr._make._frozen_setattrs try: - conf_opt = ConfOptions.from_field(field, nested=prefix) + conf_opt = ConfOptions.from_field(field, frozen=is_frozen, nested=prefix) except KeyError as e: if field.type == type: pass @@ -98,11 +103,14 @@ async def config_group(self, ctx: Context) -> None: @config_group.command(name="list") async def list_config(self, ctx: Context) -> None: """List the valid configuration options.""" - options = [] - for opt in self.config_fields.values(): - if opt.hidden: + options = {} + for table, opt in self.config_fields.items(): + if opt.hidden or opt.frozen: continue - options.append( + + # we want to merge items from the same config table so they are on the same table + key = table.rsplit(".", 1)[0] + options[key] = options.get(key, "") + ( "\n".join( [ f"**{string.capwords(opt.canconical_name or opt.name)}**", @@ -110,12 +118,12 @@ async def list_config(self, ctx: Context) -> None: if opt.default is not None else "Required. There is no default value for this option.", f"{opt.description}", - f" {opt.extended_description}" if opt.extended_description else "", + f" {opt.extended_description}\n" if opt.extended_description else "", ] ) ) - await ButtonPaginator.paginate(options, ctx.message) + await ButtonPaginator.paginate(options.values(), ctx.message) def setup(bot: ModmailBot) -> None: From 249fa5eb65209b41c3b2776886297f4cbeeea1da Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 10:37:19 -0400 Subject: [PATCH 37/76] feat: rudimentary setting and getting configuration status Signed-off-by: onerandomusername --- modmail/extensions/configuration_manager.py | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 517c06e3..8e76e28e 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,4 +1,5 @@ import logging +import operator import string import typing @@ -33,6 +34,11 @@ class ConfOptions: _type: type + metadata: dict + + modmail_metadata: config.ConfigMetadata + + _field: attr.Attribute = None nested: str = None frozen: bool = False @@ -44,9 +50,13 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str kw["name"] = field.name kw["type"] = field.type + kw["metadata"] = field.metadata + kw["field"] = field + kw["frozen"] = field.on_setattr is attr.setters.frozen or frozen meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] + kw[config.METADATA_TABLE] = meta kw["description"] = meta.description kw["canconical_name"] = meta.canconical_name kw["extended_description"] = meta.extended_description @@ -125,6 +135,35 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message) + @config_group.command(name="set", aliases=("edit",)) + async def modify_config(self, ctx: Context, option: str, value: str) -> None: + """Modify an existing configuration value.""" + if option not in self.config_fields: + raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") + meta = self.config_fields[option] + + if meta.frozen: + await ctx.send("Can't modify this value.") + return + + if meta.modmail_metadata.discord_converter is not None: + value = await meta.modmail_metadata.discord_converter().convert(ctx, value) + elif meta._field.converter: + value = meta._field.converter(value) + get_value = operator.attrgetter(option.rsplit(".", -1)[0]) + setattr(get_value(self.bot.config.user), option.rsplit(".", -1)[-1], value) + await ctx.message.reply("ok.") + + @config_group.command(name="get", aliases=("show",)) + async def get_config(self, ctx: Context, option: str) -> None: + """Modify an existing configuration value.""" + if option not in self.config_fields: + raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") + + get_value = operator.attrgetter(option) + value = get_value(self.bot.config.user) + await ctx.send(f"{option}: `{value}`") + def setup(bot: ModmailBot) -> None: """Load the ConfigurationManager cog.""" From a3402a8c399e8a4f2b58d62203c55a811b1767c9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 22 Sep 2021 18:07:46 -0400 Subject: [PATCH 38/76] refactor getvalue Signed-off-by: onerandomusername --- modmail/extensions/configuration_manager.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 8e76e28e..542cfadb 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -150,8 +150,15 @@ async def modify_config(self, ctx: Context, option: str, value: str) -> None: value = await meta.modmail_metadata.discord_converter().convert(ctx, value) elif meta._field.converter: value = meta._field.converter(value) - get_value = operator.attrgetter(option.rsplit(".", -1)[0]) - setattr(get_value(self.bot.config.user), option.rsplit(".", -1)[-1], value) + + try: + root, name = option.rsplit(".", -1) + except ValueError: + root = "" + name = option + + setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + await ctx.message.reply("ok.") @config_group.command(name="get", aliases=("show",)) From 21a46d4e69ba4713a53b8ca20d95729fffea7313 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 24 Sep 2021 06:47:30 -0400 Subject: [PATCH 39/76] fix: don't load the .env file to the environment Signed-off-by: onerandomusername --- modmail/config.py | 6 ++++-- tests/modmail/test_config.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 52c6b572..f058284b 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -409,8 +409,10 @@ def _build_class( if env is None: if dotenv_file is not None: - dotenv.load_dotenv(dotenv_file) - env = os.environ.copy() + env = dotenv.dotenv_values(dotenv_file) + env.update(os.environ) + else: + env = os.environ.copy() # get the attributes of the provided class if defaults is None: diff --git a/tests/modmail/test_config.py b/tests/modmail/test_config.py index 2548c450..a7a580a8 100644 --- a/tests/modmail/test_config.py +++ b/tests/modmail/test_config.py @@ -53,8 +53,16 @@ def test_load_env(self, tmp_path: pathlib.Path): # we have to run this here since we may have the environment vars in our local env # but we want to ensure that they are of the above env for the test - dotenv.load_dotenv(test_env, override=True) - cfg_dict = config.load_env(test_env) + try: + os.environ.update(dotenv.dotenv_values(test_env)) + cfg_dict = config.load_env(test_env) + finally: + # clean up the loaded values + for key in dotenv.dotenv_values(test_env): + try: + del os.environ[key] + except ValueError: + pass assert token == cfg_dict["bot"]["token"] assert prefix == cfg_dict["bot"]["prefix"] From a22f51a4d1e105c711208e2cfed89d9475a6be8d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 24 Sep 2021 09:23:34 -0400 Subject: [PATCH 40/76] minor: change config pre-commit hook to exit with 1 if files edited Signed-off-by: onerandomusername --- .../export_new_config_to_default_config.py | 178 +++++++++++------- 1 file changed, 114 insertions(+), 64 deletions(-) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index d3098400..86db9b75 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -4,6 +4,7 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ import json +import os import pathlib import sys import typing @@ -14,6 +15,7 @@ import dotenv import yaml +import modmail import modmail.config @@ -22,6 +24,7 @@ APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" METADATA_TABLE = modmail.config.METADATA_TABLE +MODMAIL_DIR = pathlib.Path(modmail.__file__).parent class MetadataDict(typing.TypedDict): @@ -31,7 +34,34 @@ class MetadataDict(typing.TypedDict): required: bool -def export_default_conf() -> None: +class DidFileEdit: + """Check if a file is edited within the body of this class.""" + + def __init__(self, *files: os.PathLike): + self.files: typing.List[os.PathLike] = [] + for f in files: + self.files.append(f) + self.return_value: typing.Optional[int] = None + self.edited_files: typing.List[os.PathLike] = [] + + def __enter__(self): + self.file_contents = {} + for file in self.files: + try: + with open(file, "r") as f: + self.file_contents[file] = f.read() + except FileNotFoundError: + self.file_contents[file] = None + + def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 + for file in self.files: + with open(file, "r") as f: + contents = self.file_contents[file] + if contents != f.read(): + self.edited_files.append(file) + + +def export_default_conf() -> int: """Export default configuration as both toml and yaml to the preconfigured locations.""" default = modmail.config.get_default_config() dump: dict = modmail.config.ConfigurationSchema().dump(default) @@ -58,19 +88,29 @@ def sort_dict(d: dict) -> dict: doc.update(dump) - # toml + toml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml") + yaml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml") + + check_file = DidFileEdit(toml_file, yaml_file) - with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml"), "w") as f: - atoml.dump(doc, f) + with check_file: + with open(toml_file, "w") as f: + atoml.dump(doc, f) - # yaml - with open(MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml"), "w") as f: - f.write("# This is an autogenerated YAML document.\n") - f.write(f"# {autogen_gen_notice}\n") - yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + with open(yaml_file, "w") as f: + f.write("# This is an autogenerated YAML document.\n") + f.write(f"# {autogen_gen_notice}\n") + yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) + for file in check_file.edited_files: + print( + f"Exported new configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", + file=sys.stderr, + ) + return bool(len(check_file.edited_files)) -def export_env_and_app_json_conf() -> None: + +def export_env_and_app_json_conf() -> int: """ Exports required configuration variables to .env.template. @@ -107,57 +147,68 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada return export - # dotenv modifies currently existing files, but we want to erase the current file - ENV_EXPORT_FILE.unlink(missing_ok=True) - ENV_EXPORT_FILE.touch() - - exported = get_env_vars(default.__class__) - - with open(APP_JSON_FILE) as f: - try: - app_json: typing.Dict = json.load(f) - except Exception as e: - print( - "Oops! Please ensure the app.json file is valid json! " - "If you've made manual edits, you may want to revert them." - ) - raise e - - app_json_env = dict() - - for key, meta in exported.items(): - if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): - - dotenv.set_key( - ENV_EXPORT_FILE, - key, - meta[METADATA_TABLE].export_environment_prefill or meta["default"], - ) - - if ( - meta[METADATA_TABLE].export_to_app_json - or meta[METADATA_TABLE].export_to_env_template - or meta.get("required", False) - ): - - options = defaultdict( - str, - { - "description": meta[METADATA_TABLE].description, - "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), - }, - ) - if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: - options["value"] = value - app_json_env[key] = options - - app_json["env"] = app_json_env - with open(APP_JSON_FILE, "w") as f: - json.dump(app_json, f, indent=4) - f.write("\n") - - -def main() -> None: + check_file = DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) + + with check_file: + # dotenv modifies currently existing files, but we want to erase the current file + ENV_EXPORT_FILE.unlink(missing_ok=True) + ENV_EXPORT_FILE.touch() + + exported = get_env_vars(default.__class__) + + with open(APP_JSON_FILE) as f: + try: + app_json: typing.Dict = json.load(f) + except Exception as e: + print( + "Oops! Please ensure the app.json file is valid json! " + "If you've made manual edits, you may want to revert them.", + file=sys.stderr, + ) + raise e + + app_json_env = dict() + + for key, meta in exported.items(): + if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): + + dotenv.set_key( + ENV_EXPORT_FILE, + key, + meta[METADATA_TABLE].export_environment_prefill or meta["default"], + ) + + if ( + meta[METADATA_TABLE].export_to_app_json + or meta[METADATA_TABLE].export_to_env_template + or meta.get("required", False) + ): + + options = defaultdict( + str, + { + "description": meta[METADATA_TABLE].description, + "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), + }, + ) + if (value := meta[modmail.config.METADATA_TABLE].app_json_default) is not None: + options["value"] = value + app_json_env[key] = options + + app_json["env"] = app_json_env + with open(APP_JSON_FILE, "w") as f: + json.dump(app_json, f, indent=4) + f.write("\n") + + for file in check_file.edited_files: + print( + f"Exported new env configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", + file=sys.stderr, + ) + return bool(len(check_file.edited_files)) + + +def main() -> int: """ Exports the default configuration. @@ -168,11 +219,10 @@ def main() -> None: In addition, export to app.json when exporting .env.template. """ - export_default_conf() + exit_code = export_default_conf() - export_env_and_app_json_conf() + return export_env_and_app_json_conf() or exit_code if __name__ == "__main__": - print("Exporting configuration to default files. If they exist, overwriting their contents.") sys.exit(main()) From 1c7dd19b49ffe3e3b946d44fc46602572f8bc968 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 24 Sep 2021 23:49:02 -0400 Subject: [PATCH 41/76] fix precommit requirements Signed-off-by: onerandomusername --- .pre-commit-config.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed670ebc..a51e17b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,15 +11,17 @@ repos: files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' require_serial: true additional_dependencies: - - atoml~=1.0.3 - - attrs~=21.2.0 + # so apparently these are needed, but the versions don't have to be pinned since it uses the local env + # go figure. + - atoml + - attrs - coloredlogs - desert - discord.py - environs - - marshmallow~=3.13.0 + - marshmallow - python-dotenv - - pyyaml~=5.4.1 + - pyyaml - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 From dd0e28785ef4681cfed7bc38c2c34e35d0e9ff2a Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 25 Sep 2021 00:42:11 -0400 Subject: [PATCH 42/76] config: allow emojis to be set (needs more work) Signed-off-by: onerandomusername --- modmail/config.py | 34 +++++++++++++- modmail/extensions/configuration_manager.py | 52 +++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index f058284b..6b81f743 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -2,6 +2,7 @@ import logging import os import pathlib +import types import typing from collections import defaultdict @@ -54,6 +55,30 @@ _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] + +class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): + """ + Converts to a :class:`~discord.PartialEmoji`. + + This is done by extracting the animated flag, name and ID from the emoji. + """ + + async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: + # match = self._get_id_match(argument) or re.match( + # r"$", argument + # ) + + match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) + if match is not None: + groups = match.groupdict() + animated = bool(groups["animated"]) + emoji_id = int(groups["id"]) + name = groups["name"] + return discord.PartialEmoji(name=name, animated=animated, id=emoji_id) + + return discord.PartialEmoji(name=argument) + + # load env before we do *anything* # TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. # TODO: add the above feature to the other configuration locations too. @@ -164,6 +189,9 @@ class ConfigMetadata: # as a solution, I'm implementing a field which can provide a rich converter object, # in the style that discord.py uses. This will be called like discord py calls. discord_converter: discord.ext.commands.converter.Converter = attr.ib(default=None) + discord_converter_attribute: typing.Optional[ + types.FunctionType + ] = None # if we want an attribute off of the converted value # hidden, eg log_level # hidden values mean they do not show up in the bot configuration menu @@ -320,7 +348,8 @@ class EmojiCfg: metadata={ METADATA_TABLE: ConfigMetadata( description="This is used in most cases when the bot does a successful action.", - discord_converter=discord.ext.commands.converter.EmojiConverter, + discord_converter=BetterPartialEmojiConverter, + discord_converter_attribute=lambda x: x.id or f"{x.name}", ) }, ) @@ -330,7 +359,8 @@ class EmojiCfg: metadata={ METADATA_TABLE: ConfigMetadata( description="This is used in most cases when the bot fails an action.", - discord_converter=discord.ext.commands.converter.EmojiConverter, + discord_converter=BetterPartialEmojiConverter, + discord_converter_attribute=lambda x: x.id or f"{x.name}", ) }, ) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 542cfadb..feacbbaa 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,6 +1,7 @@ import logging import operator import string +import types import typing import attr @@ -8,6 +9,7 @@ import marshmallow from discord.ext import commands from discord.ext.commands import Context +from discord.ext.commands import converter as commands_converter from modmail import config from modmail.bot import ModmailBot @@ -20,6 +22,8 @@ logger: ModmailLogger = logging.getLogger(__name__) +KeyT = str + @attr.mutable class ConfOptions: @@ -68,13 +72,13 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str return cls(**kw) -def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing.Dict[str, ConfOptions]: +def get_all_conf_options(klass: config.ClassT, *, prefix: str = "") -> typing.Dict[str, ConfOptions]: """Get a dict of ConfOptions for a designated configuration field recursively.""" options = dict() for field in attr.fields(klass): # make conf option list if attr.has(field.type): - options.update(get_all_conf_options(field.type, prefix=field.name + ".")) + options.update(get_all_conf_options(field.type, prefix=prefix + field.name + ".")) else: is_frozen = klass.__setattr__ is attr._make._frozen_setattrs try: @@ -94,6 +98,38 @@ def get_all_conf_options(klass: config.ClassT, *, prefix: str = None) -> typing. return options +class KeyConverter(commands.Converter): + """Convert argument into a configuration key.""" + + async def convert(self, ctx: Context, arg: str) -> KeyT: + """Ensure that a key is of the valid format, allowing a user to input other formats.""" + # basically we're converting an argument to a key. + # config keys are delimited by `.`, and always lowercase, which means that we can make a few passes + # before actually trying to make any guesses. + # the problems are twofold. a: we want to suggest other keys + # and b: the interface needs to be easy to use. + + # as a partial solution for this, `/`, `-`, `.` are all valid delimiters and are converted to `.` + # depending on common problems, it is *possible* to add `_` but would require fuzzy matching over + # all of the keys since that can also be a valid character name. + + fields = get_all_conf_options(config.default().__class__) + + new_arg = "" + for c in arg.lower(): + if c in " /`": + new_arg += "." + else: + new_arg += c + + if new_arg in fields: + return new_arg + else: + raise commands.BadArgument( + f"{ctx.current_parameter.name} {arg} is not a valid configuration key." + ) + + class ConfigurationManager(ModmailCog, name="Configuration Manager"): """Manage the bot configuration.""" @@ -136,18 +172,26 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message) @config_group.command(name="set", aliases=("edit",)) - async def modify_config(self, ctx: Context, option: str, value: str) -> None: + async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" if option not in self.config_fields: raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") meta = self.config_fields[option] if meta.frozen: + # TODO: replace with responses module. await ctx.send("Can't modify this value.") return if meta.modmail_metadata.discord_converter is not None: value = await meta.modmail_metadata.discord_converter().convert(ctx, value) + if meta.modmail_metadata.discord_converter_attribute is not None: + if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): + value = meta.modmail_metadata.discord_converter_attribute(value) + if meta._type in commands_converter.CONVERTER_MAPPING: + value = await commands_converter.CONVERTER_MAPPING[meta._type]().convert(ctx, value) + if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): + value = meta.modmail_metadata.discord_converter_attribute(value) elif meta._field.converter: value = meta._field.converter(value) @@ -162,7 +206,7 @@ async def modify_config(self, ctx: Context, option: str, value: str) -> None: await ctx.message.reply("ok.") @config_group.command(name="get", aliases=("show",)) - async def get_config(self, ctx: Context, option: str) -> None: + async def get_config(self, ctx: Context, option: KeyConverter) -> None: """Modify an existing configuration value.""" if option not in self.config_fields: raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") From 76ff3cd6363711bc4acaacbc038a73b8a5717689 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 4 Oct 2021 01:15:29 -0400 Subject: [PATCH 43/76] nit: fix typo in variable name --- modmail/config.py | 8 ++++---- modmail/extensions/configuration_manager.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 6b81f743..8725d3a0 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -169,7 +169,7 @@ class ConfigMetadata: # what to refer to for the end user description: str = attr.ib() - canconical_name: str = None + canonical_name: str = None # for those configuration options where the description just won't cut it. extended_description: str = None # for the variables to export to the environment, what value should be prefilled @@ -228,7 +228,7 @@ class BotCfg: "dump_only": True, "allow_none": False, METADATA_TABLE: ConfigMetadata( - canconical_name="Bot Token", + canonical_name="Bot Token", description="Discord bot token. Required to log in to discord.", export_environment_prefill="Bot Token", extended_description="This is obtainable from https://discord.com/developers/applications", @@ -240,7 +240,7 @@ class BotCfg: default="?", metadata={ METADATA_TABLE: ConfigMetadata( - canconical_name="Command Prefix", + canonical_name="Command Prefix", description="Command prefix.", export_environment_prefill="?", app_json_default="?", @@ -257,7 +257,7 @@ class BotModeCfg: The three bot modes for the bot. Enabling some of these may enable other bot features. `production` is used internally and is always True. - `develop` enables additonal features which are useful for bot developers. + `develop` enables additional features which are useful for bot developers. `plugin_dev` enables additional commands which are useful when working with plugins. """ diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index feacbbaa..872f1259 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -32,7 +32,7 @@ class ConfOptions: default: str name: str description: str - canconical_name: str + canonical_name: str extended_description: str hidden: bool @@ -62,7 +62,7 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] kw[config.METADATA_TABLE] = meta kw["description"] = meta.description - kw["canconical_name"] = meta.canconical_name + kw["canonical_name"] = meta.canonical_name kw["extended_description"] = meta.extended_description kw["hidden"] = meta.hidden @@ -159,7 +159,7 @@ async def list_config(self, ctx: Context) -> None: options[key] = options.get(key, "") + ( "\n".join( [ - f"**{string.capwords(opt.canconical_name or opt.name)}**", + f"**{string.capwords(opt.canonical_name or opt.name)}**", f"Default: `{opt.default}`" if opt.default is not None else "Required. There is no default value for this option.", From 940942168e7dc6b4470cf761c8e8cba7f8e7465a Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 4 Oct 2021 01:34:13 -0400 Subject: [PATCH 44/76] nit: typing, formatting, and comment changes --- modmail/config.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 8725d3a0..87ef04d0 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -134,7 +134,7 @@ def convert(self, argument: typing.Union[str, int, discord.Colour]) -> discord.C raise discord.ext.commands.converter.BadColourArgument(arg) return method() - def _serialize(self, value: discord.Colour, attr: str, obj: typing.Any, **kwargs) -> discord.Colour: + def _serialize(self, value: discord.Colour, attr: str, obj: typing.Any, **kwargs) -> str: return "#" + hex(value.value)[2:].lower() def _deserialize( @@ -143,7 +143,7 @@ def _deserialize( attr: typing.Optional[str], data: typing.Optional[typing.Mapping[str, typing.Any]], **kwargs, - ) -> str: + ) -> discord.Colour: if not isinstance(value, discord.Colour): value = self.ColourConvert().convert(value) return value @@ -381,8 +381,9 @@ class Cfg: default=DevCfg(), metadata={ METADATA_TABLE: ConfigMetadata( - description="Developer configuration. " - "Only change these values if you know what you're doing.", + description=( + "Developer configuration. Only change these values if you know what you're doing." + ), hidden=True, ) }, @@ -432,7 +433,7 @@ def _build_class( Defaults to getting the environment variables with dotenv. Also can parse from a provided dictionary of environment variables. - If defaults is provided, uses a value from there if the environment variable is not set or is None. + If `defaults` is provided, uses a value from there if the environment variable is not set or is None. """ if env_prefix is None: env_prefix = ENV_PREFIX @@ -464,9 +465,7 @@ def _build_class( else: kw[var.name] = env.get(env_prefix + var.name.upper(), None) if kw[var.name] is None: - if defaults is not None and ( - (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing - ): + if (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing: kw[var.name] = defaults[var.name] elif var.default is not attr.NOTHING: # check for var default kw[var.name] = var.default From 627e650a520b9c30e992d0605f3f2bef348ffcd6 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 4 Oct 2021 02:08:18 -0400 Subject: [PATCH 45/76] fix: make errors more helpful if config dependencies are not installed --- modmail/config.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 87ef04d0..7ec1adaa 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -6,7 +6,6 @@ import typing from collections import defaultdict -import atoml import attr import desert import discord @@ -18,9 +17,13 @@ import marshmallow.validate +try: + import atoml +except ModuleNotFoundError: # pragma: nocover + atoml = None try: import yaml -except ImportError: # pragma: nocover +except ModuleNotFoundError: # pragma: nocover yaml = None __all__ = [ @@ -577,6 +580,13 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: Supported file types are .toml or .yaml """ + + def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn: + raise CfgLoadError( + f"The required dependency for reading {file_type} configuration files is not installed. " + f"Please install {dependency or file_type} to allow reading these files." + ) + if len(files) == 0: files = USER_CONFIG_FILES @@ -588,17 +598,18 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: # file does not exist continue - if file.suffix == ".toml" and atoml is not None: + if file.suffix == ".toml": + if atoml is None: + raise_missing_dep("toml", "atoml") loaded_config_dict = load_toml(file) break - elif file.suffix == ".yaml" and yaml is not None: + elif file.suffix == ".yaml": + if yaml is None: + raise_missing_dep("yaml", "pyyaml") loaded_config_dict = load_yaml(file) break else: - raise Exception( - "Provided configuration file is not of a supported type or " - "the required dependencies are not installed." - ) + raise CfgLoadError("Provided configuration file is not of a supported type.") if should_load_env: loaded_config_dict = load_env(existing_cfg_dict=loaded_config_dict) From 94b14e02b930c0a6ed15e9aa1a2e48bc6270911d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 7 Nov 2021 01:40:54 -0400 Subject: [PATCH 46/76] scripts: show diff when exporting config edits files --- .../export_new_config_to_default_config.py | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 86db9b75..9edef536 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -3,6 +3,7 @@ This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ +import difflib import json import os import pathlib @@ -42,23 +43,29 @@ def __init__(self, *files: os.PathLike): for f in files: self.files.append(f) self.return_value: typing.Optional[int] = None - self.edited_files: typing.List[os.PathLike] = [] + self.edited_files: typing.Dict[os.PathLike] = dict() def __enter__(self): self.file_contents = {} for file in self.files: try: with open(file, "r") as f: - self.file_contents[file] = f.read() + self.file_contents[file] = f.readlines() except FileNotFoundError: self.file_contents[file] = None + return self def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 for file in self.files: with open(file, "r") as f: - contents = self.file_contents[file] - if contents != f.read(): - self.edited_files.append(file) + original_contents = self.file_contents[file] + new_contents = f.readlines() + if original_contents != new_contents: + # construct a diff + diff = difflib.unified_diff( + original_contents, new_contents, fromfile="before", tofile="after" + ) + self.edited_files[file] = diff def export_default_conf() -> int: @@ -91,9 +98,7 @@ def sort_dict(d: dict) -> dict: toml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml") yaml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".yaml") - check_file = DidFileEdit(toml_file, yaml_file) - - with check_file: + with DidFileEdit(toml_file, yaml_file) as check_file: with open(toml_file, "w") as f: atoml.dump(doc, f) @@ -102,11 +107,16 @@ def sort_dict(d: dict) -> dict: f.write(f"# {autogen_gen_notice}\n") yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) - for file in check_file.edited_files: + for file, diff in check_file.edited_files.items(): print( f"Exported new configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) + try: + print("".join(diff)) + except TypeError: + print("No diff to show.") + print() return bool(len(check_file.edited_files)) @@ -147,9 +157,7 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada return export - check_file = DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) - - with check_file: + with DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) as check_file: # dotenv modifies currently existing files, but we want to erase the current file ENV_EXPORT_FILE.unlink(missing_ok=True) ENV_EXPORT_FILE.touch() @@ -200,11 +208,16 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada json.dump(app_json, f, indent=4) f.write("\n") - for file in check_file.edited_files: + for file, diff in check_file.edited_files.items(): print( f"Exported new env configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) + try: + print("".join(diff)) + except TypeError: + print("No diff to show.") + print() return bool(len(check_file.edited_files)) From 4117dccf4c66cca6947722e2bf0e1a2ae4499dc4 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 7 Nov 2021 17:12:02 -0500 Subject: [PATCH 47/76] fix: no more weird new lines --- modmail/default_config.toml | 3 +-- modmail/default_config.yaml | 2 +- .../export_new_config_to_default_config.py | 20 ++++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/modmail/default_config.toml b/modmail/default_config.toml index a3202156..d77c245e 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,6 +1,5 @@ # This is an autogenerated TOML document. -# Directly run scripts/export_new_config_to_default_config.py to generate. - +# Run scripts/export_new_config_to_default_config.py as a module to generate. [bot] prefix = "?" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index de7c70cb..fd8571f1 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -1,5 +1,5 @@ # This is an autogenerated YAML document. -# Directly run scripts/export_new_config_to_default_config.py to generate. +# Run scripts/export_new_config_to_default_config.py as a module to generate. bot: prefix: '?' colours: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 9edef536..c66112f8 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -8,6 +8,7 @@ import os import pathlib import sys +import textwrap import typing from collections import defaultdict @@ -27,6 +28,15 @@ METADATA_TABLE = modmail.config.METADATA_TABLE MODMAIL_DIR = pathlib.Path(modmail.__file__).parent +# fmt: off +MESSAGE = textwrap.indent(textwrap.dedent( + f""" + This is an autogenerated {{file_type}} document. + Run scripts/{__file__.rsplit('/',1)[-1]!s} as a module to generate. + """ +), "# ").strip() + "\n" +# fmt: on + class MetadataDict(typing.TypedDict): """Typed metadata. This has a possible risk given that the modmail_metadata variable is defined.""" @@ -87,12 +97,7 @@ def sort_dict(d: dict) -> dict: return sorted_dict dump = sort_dict(dump) - autogen_gen_notice = f"Directly run scripts/{__file__.rsplit('/',1)[-1]!s} to generate." doc = atoml.document() - doc.add(atoml.comment("This is an autogenerated TOML document.")) - doc.add(atoml.comment(autogen_gen_notice)) - doc.add(atoml.nl()) - doc.update(dump) toml_file = MODMAIL_CONFIG_DIR / (modmail.config.AUTO_GEN_FILE_NAME + ".toml") @@ -100,11 +105,12 @@ def sort_dict(d: dict) -> dict: with DidFileEdit(toml_file, yaml_file) as check_file: with open(toml_file, "w") as f: + f.write(MESSAGE.format(file_type="TOML")) + f.write("\n") atoml.dump(doc, f) with open(yaml_file, "w") as f: - f.write("# This is an autogenerated YAML document.\n") - f.write(f"# {autogen_gen_notice}\n") + f.write(MESSAGE.format(file_type="YAML")) yaml.dump(dump, f, indent=4, Dumper=yaml.SafeDumper) for file, diff in check_file.edited_files.items(): From 003177e71d74f79d50840b75382cca83bf1008eb Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 19:51:39 -0500 Subject: [PATCH 48/76] minor(config): touch up the visual UI --- modmail/extensions/configuration_manager.py | 48 ++++++++++++--------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 872f1259..076d2596 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -6,6 +6,7 @@ import attr import attr._make +import discord import marshmallow from discord.ext import commands from discord.ext.commands import Context @@ -14,6 +15,7 @@ from modmail import config from modmail.bot import ModmailBot from modmail.log import ModmailLogger +from modmail.utils import responses from modmail.utils.cogs import ExtMetadata, ModmailCog from modmail.utils.pagination import ButtonPaginator @@ -48,7 +50,7 @@ class ConfOptions: @classmethod def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str = None): - """Create a ConfOptions from a attr.Attribute.""" + """Create a ConfOptions from an attr.Attribute.""" kw = {} kw["default"] = field.default if field.default is not marshmallow.missing else None kw["name"] = field.name @@ -126,7 +128,7 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: return new_arg else: raise commands.BadArgument( - f"{ctx.current_parameter.name} {arg} is not a valid configuration key." + f"{ctx.current_parameter.name.capitalize()} `{arg}` is not a valid configuration key." ) @@ -150,26 +152,30 @@ async def config_group(self, ctx: Context) -> None: async def list_config(self, ctx: Context) -> None: """List the valid configuration options.""" options = {} + + embed = discord.Embed(title="Configuration Options") for table, opt in self.config_fields.items(): + # TODO: add flag to skip this check if opt.hidden or opt.frozen: + # bot refuses to modify a hidden value + # and bot cannot modify a frozen value continue - # we want to merge items from the same config table so they are on the same table + # we want to merge items from the same config table so they display on the same pagination page + # for example, all emoji configuration options will not be split up key = table.rsplit(".", 1)[0] - options[key] = options.get(key, "") + ( - "\n".join( - [ - f"**{string.capwords(opt.canonical_name or opt.name)}**", - f"Default: `{opt.default}`" - if opt.default is not None - else "Required. There is no default value for this option.", - f"{opt.description}", - f" {opt.extended_description}\n" if opt.extended_description else "", - ] - ) - ) + if options.get(key) is None: + options[key] = f"**__{string.capwords(key)} config__**\n" + + name = opt.canonical_name or opt.name + default = f"Default: `{opt.default}`" if opt.default is not None else "Required." + description = opt.description + if opt.extended_description: + description += "\n" + opt.extended_description - await ButtonPaginator.paginate(options.values(), ctx.message) + options[key] += "\n".join([f"**{name}**", default, description]).strip() + "\n" + + await ButtonPaginator.paginate(options.values(), ctx.message, embed=embed) @config_group.command(name="set", aliases=("edit",)) async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: @@ -179,8 +185,8 @@ async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> meta = self.config_fields[option] if meta.frozen: - # TODO: replace with responses module. - await ctx.send("Can't modify this value.") + + await responses.send_negatory_response(ctx, f"Unable to modify {option}.") return if meta.modmail_metadata.discord_converter is not None: @@ -195,15 +201,15 @@ async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> elif meta._field.converter: value = meta._field.converter(value) - try: + if "." in option: root, name = option.rsplit(".", -1) - except ValueError: + else: root = "" name = option setattr(operator.attrgetter(root)(self.bot.config.user), name, value) - await ctx.message.reply("ok.") + await responses.send_positive_response(ctx, f"Successfully set `{option}` to `{value}`.") @config_group.command(name="get", aliases=("show",)) async def get_config(self, ctx: Context, option: KeyConverter) -> None: From b4e0b7d778614a4ac631b1735473e4df37b2b4ed Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 20:18:54 -0500 Subject: [PATCH 49/76] fix: update prefix when configured prefix is updated --- modmail/bot.py | 11 ++++++++++- modmail/config.py | 6 ++++++ modmail/default_config.toml | 1 + modmail/default_config.yaml | 1 + 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modmail/bot.py b/modmail/bot.py index 19f7039e..cb6dc84c 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -48,7 +48,7 @@ def __init__(self, **kwargs): activity = Activity(type=discord.ActivityType.listening, name="users dming me!") # listen to messages mentioning the bot or matching the prefix # ! NOTE: This needs to use the configuration system to get the prefix from the db once it exists. - prefix = commands.when_mentioned_or(self.config.user.bot.prefix) + prefix = self.determine_prefix # allow only user mentions by default. # ! NOTE: This may change in the future to allow roles as well allowed_mentions = AllowedMentions(everyone=False, users=True, roles=False, replied_user=True) @@ -65,6 +65,15 @@ def __init__(self, **kwargs): **kwargs, ) + @staticmethod + async def determine_prefix(bot: "ModmailBot", message: discord.Message) -> t.List[str]: + """Dynamically get the updated prefix on every command.""" + prefixes = [] + if bot.config.user.bot.prefix_when_mentioned: + prefixes.extend(commands.when_mentioned(bot, message)) + prefixes.append(bot.config.user.bot.prefix) + return prefixes + async def start(self, token: str, reconnect: bool = True) -> None: """ Start the bot. diff --git a/modmail/config.py b/modmail/config.py index 7ec1adaa..7a0f24b4 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -252,6 +252,12 @@ class BotCfg: ) }, ) + prefix_when_mentioned: bool = attr.ib( + default=True, + metadata={ + METADATA_TABLE: ConfigMetadata(description="Use the bot mention as a prefix."), + }, + ) @attr.s(auto_attribs=True, slots=True, frozen=True) diff --git a/modmail/default_config.toml b/modmail/default_config.toml index d77c245e..b1bf0b41 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -3,6 +3,7 @@ [bot] prefix = "?" +prefix_when_mentioned = true [colours] base_embed_color = "#7289da" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index fd8571f1..66e852b8 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -2,6 +2,7 @@ # Run scripts/export_new_config_to_default_config.py as a module to generate. bot: prefix: '?' + prefix_when_mentioned: true colours: base_embed_color: '#7289da' dev: From d644a96184dcbde88fea6a69a06ea3a36d355b7d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 23:45:18 -0500 Subject: [PATCH 50/76] minor fixes to export config script --- .pre-commit-config.yaml | 1 + modmail/default_config.toml | 2 +- modmail/default_config.yaml | 2 +- .../export_new_config_to_default_config.py | 64 +++++++++++++------ 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da13a94d..41e0d9ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,7 @@ repos: # go figure. - atoml - attrs + - click - coloredlogs - desert - discord.py diff --git a/modmail/default_config.toml b/modmail/default_config.toml index b1bf0b41..56f16779 100644 --- a/modmail/default_config.toml +++ b/modmail/default_config.toml @@ -1,5 +1,5 @@ # This is an autogenerated TOML document. -# Run scripts/export_new_config_to_default_config.py as a module to generate. +# Run module 'scripts.export_new_config_to_default_config' to generate. [bot] prefix = "?" diff --git a/modmail/default_config.yaml b/modmail/default_config.yaml index 66e852b8..eb88ad23 100644 --- a/modmail/default_config.yaml +++ b/modmail/default_config.yaml @@ -1,5 +1,5 @@ # This is an autogenerated YAML document. -# Run scripts/export_new_config_to_default_config.py as a module to generate. +# Run module 'scripts.export_new_config_to_default_config' to generate. bot: prefix: '?' prefix_when_mentioned: true diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index c66112f8..1190000a 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -1,7 +1,7 @@ """ Exports the configuration to the configuration default files. -This is intented to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. +This is intended to be used as a local pre-commit hook, which runs if the modmail/config.py file is changed. """ import difflib import json @@ -14,9 +14,19 @@ import atoml import attr +import click import dotenv import yaml + +try: + import pygments +except ModuleNotFoundError: + pygments = None +else: + from pygments.lexers.diff import DiffLexer + from pygments.formatters import Terminal256Formatter + import modmail import modmail.config @@ -28,11 +38,23 @@ METADATA_TABLE = modmail.config.METADATA_TABLE MODMAIL_DIR = pathlib.Path(modmail.__file__).parent + +def get_name(path: str) -> str: + """Get the script module name of the provided file path.""" + p = pathlib.Path(__file__) + p = p.relative_to(MODMAIL_DIR.parent) + name = str(p) + if name.endswith(p.suffix): + name = name[: -len(p.suffix)] + name = name.replace("/", ".") + return name + + # fmt: off MESSAGE = textwrap.indent(textwrap.dedent( f""" This is an autogenerated {{file_type}} document. - Run scripts/{__file__.rsplit('/',1)[-1]!s} as a module to generate. + Run module '{get_name(__file__)}' to generate. """ ), "# ").strip() + "\n" # fmt: on @@ -75,6 +97,9 @@ def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 diff = difflib.unified_diff( original_contents, new_contents, fromfile="before", tofile="after" ) + diff = "".join(diff) + if diff is not None and pygments is not None: + diff = pygments.highlight(diff, DiffLexer(), Terminal256Formatter()) self.edited_files[file] = diff @@ -118,10 +143,10 @@ def sort_dict(d: dict) -> dict: f"Exported new configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) - try: - print("".join(diff)) - except TypeError: - print("No diff to show.") + if diff is not None: + click.echo(diff) + else: + click.echo("No diff to show.") print() return bool(len(check_file.edited_files)) @@ -145,11 +170,11 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada env_prefix = modmail.config.ENV_PREFIX # exact name, default value - export: typing.Dict[str, MetadataDict] = dict() # any missing required vars provide as missing + export: typing.Dict[str, MetadataDict] = dict() # any missing required vars provide a sentinel for var in attr.fields(klass): if attr.has(var.type): - # var is an attrs class too, run this on it. + # var is an attrs class too, recurse over it export.update( get_env_vars( var.type, @@ -164,12 +189,7 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada return export with DidFileEdit(ENV_EXPORT_FILE, APP_JSON_FILE) as check_file: - # dotenv modifies currently existing files, but we want to erase the current file - ENV_EXPORT_FILE.unlink(missing_ok=True) - ENV_EXPORT_FILE.touch() - - exported = get_env_vars(default.__class__) - + # parse the app_json file before clearing the ENV file with open(APP_JSON_FILE) as f: try: app_json: typing.Dict = json.load(f) @@ -181,9 +201,17 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada ) raise e + # dotenv modifies currently existing files, but we want to erase the current file + # to ensure that there are no extra environment variables + ENV_EXPORT_FILE.unlink(missing_ok=True) + ENV_EXPORT_FILE.touch() + + exported = get_env_vars(default.__class__) + app_json_env = dict() for key, meta in exported.items(): + # if the value is required, or explicity asks to be exported, then we want to export it if meta[METADATA_TABLE].export_to_env_template or meta.get("required", False): dotenv.set_key( @@ -219,10 +247,10 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada f"Exported new env configuration to {pathlib.Path(file).relative_to(MODMAIL_DIR.parent)}.", file=sys.stderr, ) - try: - print("".join(diff)) - except TypeError: - print("No diff to show.") + if diff is not None: + click.echo(diff) + else: + click.echo("No diff to show.") print() return bool(len(check_file.edited_files)) From 52a339d2799b84c2edd61883ade1c75207fca332 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 14 Nov 2021 23:46:50 -0500 Subject: [PATCH 51/76] nit: use `dictionary` instead of `dit` to not look like a typo --- modmail/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 7a0f24b4..35a8eea4 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -557,15 +557,15 @@ def load_yaml(path: os.PathLike) -> dict: DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) -def _remove_extra_values(klass: type, dit: DictT) -> DictT: +def _remove_extra_values(klass: type, dictionary: DictT) -> DictT: """ Remove extra values from the provided dict which don't fit into the provided klass recursively. klass must be an attr.s class. """ fields = attr.fields_dict(klass) - cleared_dict = dit.copy() - for k in dit: + cleared_dict = dictionary.copy() + for k in dictionary: if k not in fields: del cleared_dict[k] elif isinstance(cleared_dict[k], dict): From 0937a13239b095d424abd3445ccb382223d69847 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 00:06:35 -0500 Subject: [PATCH 52/76] config: rename .env.template to template.env --- .gitignore | 1 + .pre-commit-config.yaml | 2 +- docs/contributing.md | 2 +- scripts/export_new_config_to_default_config.py | 12 ++++++++---- .env.template => template.env | 0 5 files changed, 11 insertions(+), 6 deletions(-) rename .env.template => template.env (100%) diff --git a/.gitignore b/.gitignore index d52be37a..f3bddfd2 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,7 @@ celerybeat.pid # Environments .env +!template.env .venv env/ venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41e0d9ec..a4fe9d2b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: name: Export default configuration language: python entry: poetry run python -m scripts.export_new_config_to_default_config - files: '(app\.json|\.env\.template|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' + files: '(app\.json|template\.env|modmail\/(config\.py|default_config(\.toml|\.yaml)))$' require_serial: true additional_dependencies: # so apparently these are needed, but the versions don't have to be pinned since it uses the local env diff --git a/docs/contributing.md b/docs/contributing.md index 4f351a13..5ade7573 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -181,7 +181,7 @@ $ poetry install 2. Set the modmail bot prefix in `bot.prefix`. 3. In case you are a contributor set `dev.mode.plugin_dev` and `dev.mode.develop` to `true`. The `develop` variable enables the developer bot extensions and `plugin_dev` enables plugin-developer friendly bot extensions. 4. Create a text file named `.env` in your project root (that's the base folder of your repository): - - You can also copy the `.env.template` file to `.env` + - You can also copy the `template.env` file to `.env` !!!note The entire file name is literally `.env` diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 1190000a..620b0e4c 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -32,7 +32,7 @@ MODMAIL_CONFIG_DIR = pathlib.Path(modmail.config.__file__).parent -ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / ".env.template" +ENV_EXPORT_FILE = MODMAIL_CONFIG_DIR.parent / "template.env" APP_JSON_FILE = MODMAIL_CONFIG_DIR.parent / "app.json" METADATA_TABLE = modmail.config.METADATA_TABLE @@ -97,9 +97,13 @@ def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 diff = difflib.unified_diff( original_contents, new_contents, fromfile="before", tofile="after" ) - diff = "".join(diff) - if diff is not None and pygments is not None: - diff = pygments.highlight(diff, DiffLexer(), Terminal256Formatter()) + try: + diff = "".join(diff) + except TypeError: + diff = None + else: + if pygments is not None: + diff = pygments.highlight(diff, DiffLexer(), Terminal256Formatter()) self.edited_files[file] = diff diff --git a/.env.template b/template.env similarity index 100% rename from .env.template rename to template.env From 3c60d4c4c76c7c11825e1691c782a07ec0fcc8d8 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 00:17:06 -0500 Subject: [PATCH 53/76] config(docs): document configuration loading priority --- docs/contributing.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/contributing.md b/docs/contributing.md index 5ade7573..94f46f06 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -188,6 +188,19 @@ $ poetry install 5. Open the file with any text editor and write the bot token to the files in this format: `MODMAIL_BOT_TOKEN="my_token"`. + +Of the several supported configuration sources, they are loaded in a specific priority. In decreasing priority: +- `os.environ` +- .env +- modmail_config.yaml (if PyYaml is installed and the file exists) +- modmail_config.toml (if the above file does not exist) +- defaults + +Internally, the actual parsing order may not match the above, +but the end configuration object will have the above priorty. + + + ### Run The Project To run the project, use the (below) in the project root. From 5ad572d7c76569780e62c5d4c5e260b0a6694151 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 00:45:03 -0500 Subject: [PATCH 54/76] minor: touch up comments and flatten branches --- modmail/config.py | 51 ++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 35a8eea4..ad60691c 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -67,10 +67,7 @@ class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter) """ async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: - # match = self._get_id_match(argument) or re.match( - # r"$", argument - # ) - + """Convert a provided argument into an emoji object.""" match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) if match is not None: groups = match.groupdict() @@ -196,8 +193,7 @@ class ConfigMetadata: types.FunctionType ] = None # if we want an attribute off of the converted value - # hidden, eg log_level - # hidden values mean they do not show up in the bot configuration menu + # hidden values do not show up in the bot configuration menu hidden: bool = False @description.validator @@ -447,13 +443,15 @@ def _build_class( if env_prefix is None: env_prefix = ENV_PREFIX - if env is None: - if dotenv_file is not None: - env = dotenv.dotenv_values(dotenv_file) - env.update(os.environ) - else: - env = os.environ.copy() - + # while dotenv has methods to load the .env into the environment, we don't want that + # that would mean that all of the .env variables would be loaded in to the env + # we can't assume everything in .env is for modmail + # so we load it into our own dictonary and parse from that + env = {} + if dotenv_file is not None: + env.update(dotenv.dotenv_values(dotenv_file)) + env.update(os.environ.copy()) + dotenv.load_dotenv() # get the attributes of the provided class if defaults is None: defaults = defaultdict(lambda: None) @@ -464,22 +462,29 @@ def _build_class( for var in attr.fields(klass): if attr.has(var.type): - # var is an attrs class too + # var is an attrs class too, so recurse over it kw[var.name] = _build_class( var.type, env=env, env_prefix=env_prefix + var.name.upper() + "_", defaults=defaults.get(var.name, None), ) - else: - kw[var.name] = env.get(env_prefix + var.name.upper(), None) - if kw[var.name] is None: - if (defa := defaults.get(var.name, None)) is not None and defa is not marshmallow.missing: - kw[var.name] = defaults[var.name] - elif var.default is not attr.NOTHING: # check for var default - kw[var.name] = var.default - else: - del kw[var.name] + continue + + kw[var.name] = env.get(env_prefix + var.name.upper(), None) + + if kw[var.name] is not None: + continue + + if var.name in defaults and defaults[var.name] not in [None, marshmallow.missing]: + kw[var.name] = defaults[var.name] + continue + + if var.default is not attr.NOTHING: # check for var default + kw[var.name] = var.default + continue + + del kw[var.name] return klass(**kw) From 8bc7c2771885ba09b6fff115d68dcbb2274aa063 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 20:07:19 -0500 Subject: [PATCH 55/76] fix: use commands.run_converters() in the configuration manager ensures that we use the cache and special conversion methods to convert variables this means that all of the varaible conversion is handled by discord.py all conversion errors are propagated outwards and caught by our error handler --- modmail/extensions/configuration_manager.py | 55 ++++++++++----------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 076d2596..7b212bb0 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,3 +1,4 @@ +import inspect import logging import operator import string @@ -10,7 +11,6 @@ import marshmallow from discord.ext import commands from discord.ext.commands import Context -from discord.ext.commands import converter as commands_converter from modmail import config from modmail.bot import ModmailBot @@ -128,7 +128,8 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: return new_arg else: raise commands.BadArgument( - f"{ctx.current_parameter.name.capitalize()} `{arg}` is not a valid configuration key." + f"{ctx.current_parameter.name.capitalize()} `{arg}` is not a valid configuration key.\n" + f"{ctx.current_parameter.name.capitalize()} must be in {', '.join(fields.keys())}" ) @@ -180,46 +181,42 @@ async def list_config(self, ctx: Context) -> None: @config_group.command(name="set", aliases=("edit",)) async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" - if option not in self.config_fields: - raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") - meta = self.config_fields[option] + metadata = self.config_fields[option] - if meta.frozen: - - await responses.send_negatory_response(ctx, f"Unable to modify {option}.") + if metadata.frozen: + await responses.send_negatory_response( + ctx, f"Unable to modify `{option}` as it is frozen and cannot be edited during runtime." + ) return - if meta.modmail_metadata.discord_converter is not None: - value = await meta.modmail_metadata.discord_converter().convert(ctx, value) - if meta.modmail_metadata.discord_converter_attribute is not None: - if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): - value = meta.modmail_metadata.discord_converter_attribute(value) - if meta._type in commands_converter.CONVERTER_MAPPING: - value = await commands_converter.CONVERTER_MAPPING[meta._type]().convert(ctx, value) - if isinstance(meta.modmail_metadata.discord_converter_attribute, types.FunctionType): - value = meta.modmail_metadata.discord_converter_attribute(value) - elif meta._field.converter: - value = meta._field.converter(value) + # since we are in the bot, we are able to run commands.run_converters() + # we have a context object, and several other objects, so this should be able to work + param = inspect.Parameter("value", 1, default=metadata.default, annotation=metadata._type) + converted_result = await commands.run_converters(ctx, metadata._type, value, param) + + discord_converter_attribute = metadata.modmail_metadata.discord_converter_attribute + if isinstance(discord_converter_attribute, types.FunctionType): + converted_result = discord_converter_attribute(converted_result) if "." in option: - root, name = option.rsplit(".", -1) + root, name = option.rsplit(".", 1) else: root = "" name = option - setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + setattr(operator.attrgetter(root)(self.bot.config.user), name, converted_result) - await responses.send_positive_response(ctx, f"Successfully set `{option}` to `{value}`.") + if converted_result == value: + response = f"Successfully set `{option}` to `{value}`." + else: + response = f"Successfully set `{option}` to `{converted_result}` (converted from `{value}`)" + await responses.send_positive_response(ctx, response) @config_group.command(name="get", aliases=("show",)) async def get_config(self, ctx: Context, option: KeyConverter) -> None: - """Modify an existing configuration value.""" - if option not in self.config_fields: - raise commands.BadArgument(f"Option must be in {', '.join(self.config_fields.keys())}") - - get_value = operator.attrgetter(option) - value = get_value(self.bot.config.user) - await ctx.send(f"{option}: `{value}`") + """Display an existing configuration value.""" + value = operator.attrgetter(option)(self.bot.config.user) + await responses.send_general_response(ctx, f"value: `{value}`", embed=discord.Embed(title=option)) def setup(bot: ModmailBot) -> None: From 263ad6bd50d7e824ddd02a7ac010c0a1b695379e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 20:10:30 -0500 Subject: [PATCH 56/76] config-manager: add set-default command to reset to default --- modmail/extensions/configuration_manager.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 7b212bb0..5bb7e57d 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -178,6 +178,31 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message, embed=embed) + @config_group.command(name="set_default", aliases=("set-default",)) + async def set_default(self, ctx: Context, option: KeyConverter) -> None: + """Reset the provided configuration value to the default.""" + if "." in option: + root, name = option.rsplit(".", 1) + else: + root = "" + name = option + value = operator.attrgetter(option)(self.bot.config.default) + if value in [marshmallow.missing, attr.NOTHING]: + await responses.send_negatory_response( + ctx, f"`{option}` is a required configuration variable and cannot be reset." + ) + return + try: + setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + except (attr.exceptions.FrozenAttributeError, attr.exceptions.FrozenInstanceError): + await responses.send_negatory_response( + ctx, f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." + ) + else: + await responses.send_positive_response( + ctx, f"Successfully set `{option}` to the default of `{value}`." + ) + @config_group.command(name="set", aliases=("edit",)) async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" From 0a596b4d2f55372d9aa21e3c4aefb64f08576366 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 20:39:34 -0500 Subject: [PATCH 57/76] add set default command and offload setting config to a helper method --- modmail/extensions/configuration_manager.py | 60 ++++++++++++--------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 5bb7e57d..21933400 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -27,6 +27,12 @@ KeyT = str +class UnableToModifyConfig(commands.CommandError): + """Raised when a command is unable to modify the configuration.""" + + pass + + @attr.mutable class ConfOptions: """Configuration attribute class.""" @@ -178,33 +184,45 @@ async def list_config(self, ctx: Context) -> None: await ButtonPaginator.paginate(options.values(), ctx.message, embed=embed) - @config_group.command(name="set_default", aliases=("set-default",)) - async def set_default(self, ctx: Context, option: KeyConverter) -> None: - """Reset the provided configuration value to the default.""" + _T = typing.TypeVar("_T") + + async def set_config_value(self, option: str, new_value: _T) -> typing.Tuple[str, _T]: + """ + Set the provided option to new_value. + + Raises an UnableToModifyConfig error if there is an issue + """ if "." in option: root, name = option.rsplit(".", 1) else: root = "" name = option - value = operator.attrgetter(option)(self.bot.config.default) - if value in [marshmallow.missing, attr.NOTHING]: - await responses.send_negatory_response( - ctx, f"`{option}` is a required configuration variable and cannot be reset." - ) - return + + if new_value in [marshmallow.missing, attr.NOTHING]: + raise UnableToModifyConfig( + f"`{option}` is a required configuration variable and cannot be reset." + ) from None try: - setattr(operator.attrgetter(root)(self.bot.config.user), name, value) + setattr(operator.attrgetter(root)(self.bot.config.user), name, new_value) except (attr.exceptions.FrozenAttributeError, attr.exceptions.FrozenInstanceError): - await responses.send_negatory_response( - ctx, f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." - ) + raise UnableToModifyConfig( + f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." + ) from None else: - await responses.send_positive_response( - ctx, f"Successfully set `{option}` to the default of `{value}`." - ) + return (option, new_value) + + @config_group.command(name="set_default", aliases=("set-default",)) + async def set_default(self, ctx: Context, option: KeyConverter) -> None: + """Reset the provided configuration value to the default.""" + value = operator.attrgetter(option)(self.bot.config.default) + await self.set_config_value(option, value) + + await responses.send_positive_response( + ctx, f"Successfully set `{option}` to the default of `{value}`." + ) @config_group.command(name="set", aliases=("edit",)) - async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> None: + async def modify_config_command(self, ctx: Context, option: KeyConverter, value: str) -> None: """Modify an existing configuration value.""" metadata = self.config_fields[option] @@ -223,13 +241,7 @@ async def modify_config(self, ctx: Context, option: KeyConverter, value: str) -> if isinstance(discord_converter_attribute, types.FunctionType): converted_result = discord_converter_attribute(converted_result) - if "." in option: - root, name = option.rsplit(".", 1) - else: - root = "" - name = option - - setattr(operator.attrgetter(root)(self.bot.config.user), name, converted_result) + await self.set_config_value(option, value) if converted_result == value: response = f"Successfully set `{option}` to `{value}`." From 81c08da536125913aed657ff417dee10b91714c1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 15 Nov 2021 21:00:54 -0500 Subject: [PATCH 58/76] fix: add missing character to filter, fix env loading --- modmail/config.py | 11 ++++------- modmail/extensions/configuration_manager.py | 2 +- modmail/utils/extensions.py | 3 ++- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index ad60691c..4a2146f8 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -428,7 +428,7 @@ class Config: def _build_class( klass: ClassT, env: typing.Dict[str, str] = None, - env_prefix: str = None, + env_prefix: str = ENV_PREFIX, *, dotenv_file: os.PathLike = None, defaults: typing.Dict = None, @@ -440,9 +440,6 @@ def _build_class( Also can parse from a provided dictionary of environment variables. If `defaults` is provided, uses a value from there if the environment variable is not set or is None. """ - if env_prefix is None: - env_prefix = ENV_PREFIX - # while dotenv has methods to load the .env into the environment, we don't want that # that would mean that all of the .env variables would be loaded in to the env # we can't assume everything in .env is for modmail @@ -451,7 +448,7 @@ def _build_class( if dotenv_file is not None: env.update(dotenv.dotenv_values(dotenv_file)) env.update(os.environ.copy()) - dotenv.load_dotenv() + # get the attributes of the provided class if defaults is None: defaults = defaultdict(lambda: None) @@ -503,9 +500,9 @@ def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> di if not existing_cfg_dict: existing_cfg_dict = defaultdict(_generate_default_dict) - existing_cfg_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) + new_config_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) - return existing_cfg_dict + return new_config_dict def load_toml(path: os.PathLike = None) -> defaultdict: diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 21933400..002cfeeb 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -125,7 +125,7 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: new_arg = "" for c in arg.lower(): - if c in " /`": + if c in "./-": new_arg += "." else: new_arg += c diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index b7bf7cae..5e5d47fd 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -34,8 +34,9 @@ def determine_bot_mode() -> int: The configuration system uses true/false values, so we need to turn them into an integer for bitwise. """ bot_mode = 0 + _config = config() for mode in BotModes: - if getattr(config().user.dev.mode, unqualify(str(mode)).lower(), True): + if getattr(_config.user.dev.mode, unqualify(str(mode)).lower(), True): bot_mode += mode.value return bot_mode From 196d2a8a640e342c9530b661827271007c4d16cf Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 01:17:37 -0500 Subject: [PATCH 59/76] fix: use metadata provided converter to convert objects --- modmail/config.py | 4 ++-- modmail/extensions/configuration_manager.py | 24 +++++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 4a2146f8..654b8274 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -348,7 +348,7 @@ class EmojiCfg: This was a pain to implement. """ - success: typing.Any = attr.ib( + success: str = attr.ib( default=":thumbsup:", metadata={ METADATA_TABLE: ConfigMetadata( @@ -359,7 +359,7 @@ class EmojiCfg: }, ) - failure: typing.Any = attr.ib( + failure: str = attr.ib( default=":x:", metadata={ METADATA_TABLE: ConfigMetadata( diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 002cfeeb..9c2c0a2e 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -44,11 +44,13 @@ class ConfOptions: extended_description: str hidden: bool - _type: type + type: type metadata: dict modmail_metadata: config.ConfigMetadata + discord_converter: commands.Converter + discord_converter_attribute: types.FunctionType = None _field: attr.Attribute = None nested: str = None @@ -67,12 +69,15 @@ def from_field(cls, field: attr.Attribute, *, frozen: bool = False, nested: str kw["frozen"] = field.on_setattr is attr.setters.frozen or frozen - meta: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] - kw[config.METADATA_TABLE] = meta - kw["description"] = meta.description - kw["canonical_name"] = meta.canonical_name - kw["extended_description"] = meta.extended_description - kw["hidden"] = meta.hidden + metadata_table: config.ConfigMetadata = field.metadata[config.METADATA_TABLE] + kw[config.METADATA_TABLE] = metadata_table + kw["description"] = metadata_table.description + kw["canonical_name"] = metadata_table.canonical_name + kw["extended_description"] = metadata_table.extended_description + kw["hidden"] = metadata_table.hidden + + kw["discord_converter"] = metadata_table.discord_converter + kw["discord_converter_attribute"] = metadata_table.discord_converter_attribute if nested is not None: kw["nested"] = nested @@ -234,8 +239,9 @@ async def modify_config_command(self, ctx: Context, option: KeyConverter, value: # since we are in the bot, we are able to run commands.run_converters() # we have a context object, and several other objects, so this should be able to work - param = inspect.Parameter("value", 1, default=metadata.default, annotation=metadata._type) - converted_result = await commands.run_converters(ctx, metadata._type, value, param) + annotation = metadata.discord_converter or metadata.type + param = inspect.Parameter("value", 1, default=metadata.default, annotation=annotation) + converted_result = await commands.run_converters(ctx, annotation, value, param) discord_converter_attribute = metadata.modmail_metadata.discord_converter_attribute if isinstance(discord_converter_attribute, types.FunctionType): From ebcaf9485e416619e22f5b12a2e05e4cc0e23ee9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 01:59:27 -0500 Subject: [PATCH 60/76] nits: standardize naming and make class names more self explanatory cfg, config, configuration... why so many similarily named classes??? --- modmail/config.py | 89 +++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 654b8274..3b894ba5 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -31,15 +31,14 @@ "ENV_PREFIX", "USER_CONFIG_FILE_NAME", "USER_CONFIG_FILES", - "CfgLoadError", + "ConfigLoadError", "Config", "config", "ConfigurationSchema", - "BotCfg", - "BotModeCfg", - "Cfg", - "Colours", - "DevCfg", + "BotConfig", + "BotModeConfig", + "ColourConfig", + "DeveloperConfig", "convert_to_color", "get_config", "get_default_config", @@ -80,7 +79,7 @@ async def convert(self, _: discord.ext.commands.context.Context, argument: str) # load env before we do *anything* -# TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. +# !!! TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. # TODO: add the above feature to the other configuration locations too. dotenv.load_dotenv(_CWD / ".env") @@ -90,10 +89,10 @@ def _generate_default_dict() -> defaultdict: return defaultdict(_generate_default_dict) -class CfgLoadError(Exception): +class ConfigLoadError(Exception): """Exception if the configuration failed to load from a local file.""" - ... + pass class _ColourField(marshmallow.fields.Field): @@ -157,7 +156,7 @@ def convert_to_color(col: typing.Union[str, int, discord.Colour]) -> discord.Col @attr.frozen(kw_only=True) class ConfigMetadata: """ - Cfg metadata. This is intended to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. + Config metadata. The intent is to be used on the marshmallow and attr metadata dict as 'modmail_metadata'. Nearly all of these values are optional, save for the description. All of them are keyword only. @@ -210,7 +209,7 @@ def _validate_discord_converter(self, attrib: attr.Attribute, value: typing.Any) @attr.s(auto_attribs=True, slots=True) -class BotCfg: +class BotConfig: """ Values that are configuration for the bot itself. @@ -257,7 +256,7 @@ class BotCfg: @attr.s(auto_attribs=True, slots=True, frozen=True) -class BotModeCfg: +class BotModeConfig: """ The three bot modes for the bot. Enabling some of these may enable other bot features. @@ -299,7 +298,7 @@ class BotModeCfg: @attr.s(auto_attribs=True, slots=True) -class Colours: +class ColourConfig: """ Default colors. @@ -319,10 +318,10 @@ class Colours: @attr.s(auto_attribs=True, slots=True) -class DevCfg: +class DeveloperConfig: """Developer configuration. These values should not be changed unless you know what you're doing.""" - mode: BotModeCfg = BotModeCfg() + mode: BotModeConfig = BotModeConfig() log_level: int = attr.ib( default=logging.INFO, metadata={ @@ -341,7 +340,7 @@ def _log_level_validator(self, a: attr.Attribute, value: int) -> None: @attr.mutable(slots=True) -class EmojiCfg: +class EmojiConfig: """ Emojis used across the entire bot. @@ -372,7 +371,7 @@ class EmojiCfg: @attr.s(auto_attribs=True, slots=True) -class Cfg: +class BaseConfig: """ Base configuration attrs class. @@ -380,10 +379,10 @@ class Cfg: we can get a clean default variable if we don't pass anything. """ - bot: BotCfg = BotCfg() - colours: Colours = Colours() - dev: DevCfg = attr.ib( - default=DevCfg(), + bot: BotConfig = BotConfig() + colours: ColourConfig = ColourConfig() + dev: DeveloperConfig = attr.ib( + default=DeveloperConfig(), metadata={ METADATA_TABLE: ConfigMetadata( description=( @@ -393,15 +392,15 @@ class Cfg: ) }, ) - emojis: EmojiCfg = EmojiCfg() + emojis: EmojiConfig = EmojiConfig() # build configuration -ConfigurationSchema = desert.schema_class(Cfg, meta={"ordered": True}) # noqa: N818 +ConfigurationSchema = desert.schema_class(BaseConfig, meta={"ordered": True}) # noqa: N818 _CACHED_CONFIG: "Config" = None -_CACHED_DEFAULT: Cfg = None +_CACHED_DEFAULT: BaseConfig = None @attr.s(auto_attribs=True, slots=True, kw_only=True) @@ -410,15 +409,15 @@ class Config: Base configuration variable. Used across the entire bot for configuration variables. Holds two variables, default and user. - Default is a Cfg instance with nothing passed. It is a default instance of Cfg. + Default is a BaseConfig instance with nothing passed. It is a default instance of BaseConfig. - User is a Cfg schema instance, generated from a combination of the defaults, + User is a BaseConfig schema instance, generated from a combination of the defaults, user provided toml, and environment variables. """ - user: Cfg + user: BaseConfig schema: marshmallow.Schema - default: Cfg = Cfg() + default: BaseConfig = BaseConfig() ClassT = typing.TypeVar("ClassT", bound=type) @@ -486,7 +485,7 @@ def _build_class( return klass(**kw) -def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> dict: +def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> dict: """ Load a configuration dictionary from the specified env file and environment variables. @@ -497,10 +496,16 @@ def load_env(env_file: os.PathLike = None, existing_cfg_dict: dict = None) -> di else: env_file = pathlib.Path(env_file) - if not existing_cfg_dict: - existing_cfg_dict = defaultdict(_generate_default_dict) + if not existing_config_dict: + existing_config_dict = defaultdict(_generate_default_dict) - new_config_dict = attr.asdict(_build_class(Cfg, dotenv_file=env_file, defaults=existing_cfg_dict)) + new_config_dict = attr.asdict( + _build_class( + BaseConfig, + dotenv_file=env_file, + defaults=existing_config_dict, + ) + ) return new_config_dict @@ -518,13 +523,13 @@ def load_toml(path: os.PathLike = None) -> defaultdict: path = pathlib.Path(path) if not path.is_file(): - raise CfgLoadError("The provided toml file path is not a valid file.") + raise ConfigLoadError("The provided toml file path is not a valid file.") try: with open(path) as f: return defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) except Exception as e: - raise CfgLoadError from e + raise ConfigLoadError from e def load_yaml(path: os.PathLike) -> dict: @@ -547,13 +552,13 @@ def load_yaml(path: os.PathLike) -> dict: ("The provided yaml config file is not a readable file.", path.is_file()), ] if errors := "\n".join(msg for msg, check in states if not check): - raise CfgLoadError(errors) + raise ConfigLoadError(errors) try: with open(path, "r") as f: return defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) except Exception as e: - raise CfgLoadError from e + raise ConfigLoadError from e DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) @@ -590,7 +595,7 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: """ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn: - raise CfgLoadError( + raise ConfigLoadError( f"The required dependency for reading {file_type} configuration files is not installed. " f"Please install {dependency or file_type} to allow reading these files." ) @@ -617,16 +622,16 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn loaded_config_dict = load_yaml(file) break else: - raise CfgLoadError("Provided configuration file is not of a supported type.") + raise ConfigLoadError("Provided configuration file is not of a supported type.") if should_load_env: - loaded_config_dict = load_env(existing_cfg_dict=loaded_config_dict) + loaded_config_dict = load_env(existing_config_dict=loaded_config_dict) # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. - loaded_config_dict = _remove_extra_values(Cfg, loaded_config_dict) + loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) @@ -644,11 +649,11 @@ def get_config() -> Config: return _CACHED_CONFIG -def get_default_config() -> Cfg: +def get_default_config() -> BaseConfig: """Get the default configuration instance of the global Config instance.""" global _CACHED_DEFAULT if _CACHED_DEFAULT is None: - _CACHED_DEFAULT = Cfg() + _CACHED_DEFAULT = BaseConfig() return _CACHED_DEFAULT From 2fd074c705f42b27acd9c0ef625656a0a566b989 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 09:18:29 -0500 Subject: [PATCH 61/76] config: make dotenv optional and better errors when missing variables --- modmail/config.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 3b894ba5..8a8dff96 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -11,12 +11,16 @@ import discord import discord.ext.commands import discord.ext.commands.converter -import dotenv import marshmallow import marshmallow.fields import marshmallow.validate +try: + import dotenv +except ModuleNotFoundError: # pragma: nocover + dotenv = None + try: import atoml except ModuleNotFoundError: # pragma: nocover @@ -57,6 +61,8 @@ _CWD / (USER_CONFIG_FILE_NAME + ".toml"), ] +logger = logging.getLogger(__name__) + class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): """ @@ -78,12 +84,6 @@ async def convert(self, _: discord.ext.commands.context.Context, argument: str) return discord.PartialEmoji(name=argument) -# load env before we do *anything* -# !!! TODO: Convert this to a function and check the parent directory too, if the CWD is within the bot. -# TODO: add the above feature to the other configuration locations too. -dotenv.load_dotenv(_CWD / ".env") - - def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" return defaultdict(_generate_default_dict) @@ -443,8 +443,12 @@ def _build_class( # that would mean that all of the .env variables would be loaded in to the env # we can't assume everything in .env is for modmail # so we load it into our own dictonary and parse from that - env = {} + if env is None: + env = {} + if dotenv_file is not None: + if dotenv is None: + raise ConfigLoadError("dotenv extra must be installed to read .env files.") env.update(dotenv.dotenv_values(dotenv_file)) env.update(os.environ.copy()) @@ -492,7 +496,9 @@ def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> All dependencies for this will always be installed. """ if env_file is None: - env_file = _CWD / ".env" + # only presume an .env file if dotenv is not None + if dotenv is not None: + env_file = _CWD / ".env" else: env_file = pathlib.Path(env_file) @@ -627,13 +633,16 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn if should_load_env: loaded_config_dict = load_env(existing_config_dict=loaded_config_dict) - # HACK remove extra keeps from the configuration dict since marshmallow doesn't know what to do with them + # HACK remove extra keys from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) - - loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) + try: + loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) + except marshmallow.ValidationError: + logger.exception("Unable to load the configuration.") + exit(1) return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) From fa02b3436bee93b34211930d830fb7b9069d31c1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 11:11:21 -0500 Subject: [PATCH 62/76] config: add logging and stabilize configuration directory --- modmail/config.py | 82 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 8a8dff96..faf5e36c 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -1,3 +1,4 @@ +import functools import inspect import logging import os @@ -15,23 +16,34 @@ import marshmallow.fields import marshmallow.validate +from modmail.log import ModmailLogger + +_log_optional_deps = logging.getLogger("modmail.optional_dependencies") +_silence = " You can silence these alerts by ignoring the `modmail.optional_dependencies` logger" try: import dotenv except ModuleNotFoundError: # pragma: nocover + _log_optional_deps.notice("dotenv was unable to be imported." + _silence) dotenv = None try: import atoml except ModuleNotFoundError: # pragma: nocover + _log_optional_deps.notice("atoml was unable to be imported." + _silence) atoml = None try: import yaml except ModuleNotFoundError: # pragma: nocover + _log_optional_deps.notice("yaml was unable to be imported." + _silence) yaml = None +del _log_optional_deps +del _silence + __all__ = [ "AUTO_GEN_FILE_NAME", + "CONFIG_DIRECTORY", "ENV_PREFIX", "USER_CONFIG_FILE_NAME", "USER_CONFIG_FILES", @@ -51,18 +63,45 @@ "load_yaml", ] -_CWD = pathlib.Path.cwd() + +logger: ModmailLogger = logging.getLogger(__name__) + + +@functools.lru_cache +def _get_config_directory() -> pathlib.Path: + """ + Return a directory where the configuration should reside. + + This is typically the current working directory, with a cavet. + + If the current directory's path includes the first parent directory of the bot + then the parent directory of the bot is looked for configuration files. + """ + import modmail + + path = pathlib.Path(modmail.__file__).parent.parent + cwd = pathlib.Path.cwd() + try: + cwd.relative_to(path) + except ValueError: + result = cwd + else: + result = path + logger.trace(f"Determined bot root directory from cwd as {result}") + logger.trace("That directory is either the cwd, or the common parent of the bot and cwd directories.") + return result + + +CONFIG_DIRECTORY = _get_config_directory() METADATA_TABLE = "modmail_metadata" ENV_PREFIX = "MODMAIL_" USER_CONFIG_FILE_NAME = "modmail_config" AUTO_GEN_FILE_NAME = "default_config" USER_CONFIG_FILES = [ - _CWD / (USER_CONFIG_FILE_NAME + ".yaml"), - _CWD / (USER_CONFIG_FILE_NAME + ".toml"), + CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml"), + CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml"), ] -logger = logging.getLogger(__name__) - class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): """ @@ -431,6 +470,7 @@ def _build_class( *, dotenv_file: os.PathLike = None, defaults: typing.Dict = None, + skip_dotenv_if_none: bool = False, ) -> ClassT: """ Create an instance of the provided klass from env vars prefixed with ENV_PREFIX and class_prefix. @@ -447,9 +487,12 @@ def _build_class( env = {} if dotenv_file is not None: - if dotenv is None: + if dotenv is not None: + env.update(dotenv.dotenv_values(dotenv_file)) + elif not skip_dotenv_if_none: raise ConfigLoadError("dotenv extra must be installed to read .env files.") - env.update(dotenv.dotenv_values(dotenv_file)) + else: + logger.info(f"{dotenv = }, but {skip_dotenv_if_none = }" f" so ignoring reading {dotenv_file = }") env.update(os.environ.copy()) # get the attributes of the provided class @@ -489,16 +532,18 @@ def _build_class( return klass(**kw) -def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> dict: +def load_env( + env_file: os.PathLike = None, existing_config_dict: dict = None, *, skip_dotenv_if_none: bool = False +) -> dict: """ Load a configuration dictionary from the specified env file and environment variables. All dependencies for this will always be installed. """ + logger.trace(f"function `load_env` called to load env_file {env_file}") + if env_file is None: - # only presume an .env file if dotenv is not None - if dotenv is not None: - env_file = _CWD / ".env" + env_file = CONFIG_DIRECTORY / ".env" else: env_file = pathlib.Path(env_file) @@ -510,6 +555,7 @@ def load_env(env_file: os.PathLike = None, existing_config_dict: dict = None) -> BaseConfig, dotenv_file=env_file, defaults=existing_config_dict, + skip_dotenv_if_none=skip_dotenv_if_none, ) ) @@ -522,8 +568,10 @@ def load_toml(path: os.PathLike = None) -> defaultdict: All dependencies for this will always be installed. """ + logger.trace(f"function `load_toml` called to load toml file {path}") + if path is None: - path = (_CWD / (USER_CONFIG_FILE_NAME + ".toml"),) + path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml"),) else: # fully resolve path path = pathlib.Path(path) @@ -547,8 +595,9 @@ def load_yaml(path: os.PathLike) -> dict: In order to keep errors at a minimum, this function checks if both pyyaml is installed and if a yaml configuration file exists before raising an exception. """ + logger.trace(f"function `load_yaml` called to load yaml file {path}") if path is None: - path = (_CWD / (USER_CONFIG_FILE_NAME + ".yaml"),) + path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml"),) else: path = pathlib.Path(path) @@ -601,10 +650,12 @@ def _load_config(*files: os.PathLike, should_load_env: bool = True) -> Config: """ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn: - raise ConfigLoadError( + msg = ( f"The required dependency for reading {file_type} configuration files is not installed. " f"Please install {dependency or file_type} to allow reading these files." ) + logger.error(msg) + raise ConfigLoadError(msg) if len(files) == 0: files = USER_CONFIG_FILES @@ -631,13 +682,14 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn raise ConfigLoadError("Provided configuration file is not of a supported type.") if should_load_env: - loaded_config_dict = load_env(existing_config_dict=loaded_config_dict) + loaded_config_dict = load_env(existing_config_dict=loaded_config_dict, skip_dotenv_if_none=True) # HACK remove extra keys from the configuration dict since marshmallow doesn't know what to do with them # CONTRARY to the marshmallow.EXCLUDE below. # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) + logger.debug("configuration loaded_config_dict prepped. Attempting to seralize...") try: loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) except marshmallow.ValidationError: From 3d1986d82bdb1d21b5965639f335fbe08631bd22 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 11:18:48 -0500 Subject: [PATCH 63/76] minor: use functools cache to cache the config --- modmail/config.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index faf5e36c..5f7b5e2d 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -438,10 +438,6 @@ class BaseConfig: ConfigurationSchema = desert.schema_class(BaseConfig, meta={"ordered": True}) # noqa: N818 -_CACHED_CONFIG: "Config" = None -_CACHED_DEFAULT: BaseConfig = None - - @attr.s(auto_attribs=True, slots=True, kw_only=True) class Config: """ @@ -698,24 +694,20 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) +@functools.lru_cache(None) def get_config() -> Config: """ Helps to try to ensure that only one instance of the Config class exists. This means that all usage of the configuration is using the same configuration class. """ - global _CACHED_CONFIG - if _CACHED_CONFIG is None: - _CACHED_CONFIG = _load_config() - return _CACHED_CONFIG + return _load_config() +@functools.lru_cache(None) def get_default_config() -> BaseConfig: """Get the default configuration instance of the global Config instance.""" - global _CACHED_DEFAULT - if _CACHED_DEFAULT is None: - _CACHED_DEFAULT = BaseConfig() - return _CACHED_DEFAULT + return BaseConfig() config = get_config From ff04a5ae2c6d8f73a231391e6c9fa848d2f9502e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 12:31:01 -0500 Subject: [PATCH 64/76] use factories to generate mutable classes and add more logging tests: mark xfail the test for default config caching --- modmail/config.py | 41 +++++++++++++++++++++++------------- tests/modmail/test_config.py | 7 ++++-- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 5f7b5e2d..1050f007 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -360,7 +360,7 @@ class ColourConfig: class DeveloperConfig: """Developer configuration. These values should not be changed unless you know what you're doing.""" - mode: BotModeConfig = BotModeConfig() + mode: BotModeConfig = attr.ib(factory=BotModeConfig) log_level: int = attr.ib( default=logging.INFO, metadata={ @@ -418,10 +418,10 @@ class BaseConfig: we can get a clean default variable if we don't pass anything. """ - bot: BotConfig = BotConfig() - colours: ColourConfig = ColourConfig() + bot: BotConfig = attr.ib(factory=BotConfig) + colours: ColourConfig = attr.ib(factory=ColourConfig) dev: DeveloperConfig = attr.ib( - default=DeveloperConfig(), + factory=DeveloperConfig, metadata={ METADATA_TABLE: ConfigMetadata( description=( @@ -438,7 +438,17 @@ class BaseConfig: ConfigurationSchema = desert.schema_class(BaseConfig, meta={"ordered": True}) # noqa: N818 -@attr.s(auto_attribs=True, slots=True, kw_only=True) +# @functools.lru_cache(None) +def get_default_config() -> BaseConfig: + """Get the default configuration instance of the BaseConfig instance.""" + # TODO: This should be a frozen instance all the way down on every level + # TODO: but I'm not sure the best way to do that, or even how to do that + # TODO: as a result, the lrucache decorator is temporarily commented out + # TODO: until we discover the best way to freeze this and all subclasses + return BaseConfig() + + +@attr.s(auto_attribs=True, slots=True, kw_only=True, frozen=True) class Config: """ Base configuration variable. Used across the entire bot for configuration variables. @@ -452,7 +462,7 @@ class Config: user: BaseConfig schema: marshmallow.Schema - default: BaseConfig = BaseConfig() + default: BaseConfig = get_default_config() ClassT = typing.TypeVar("ClassT", bound=type) @@ -577,10 +587,13 @@ def load_toml(path: os.PathLike = None) -> defaultdict: try: with open(path) as f: - return defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) + result = defaultdict(lambda: marshmallow.missing, atoml.parse(f.read()).value) except Exception as e: raise ConfigLoadError from e + logger.info(f"Successfully parsed toml config located at {path!s}") + return result + def load_yaml(path: os.PathLike) -> dict: """ @@ -607,10 +620,13 @@ def load_yaml(path: os.PathLike) -> dict: try: with open(path, "r") as f: - return defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) + result = defaultdict(lambda: marshmallow.missing, yaml.load(f.read(), Loader=yaml.SafeLoader)) except Exception as e: raise ConfigLoadError from e + logger.info(f"Successfully parsed yaml config located at {path!s}") + return result + DictT = typing.TypeVar("DictT", bound=typing.Dict[str, typing.Any]) @@ -685,12 +701,13 @@ def raise_missing_dep(file_type: str, dependency: str = None) -> typing.NoReturn # They will cause errors. # Extra configuration values are okay, we aren't trying to be strict here. loaded_config_dict = _remove_extra_values(BaseConfig, loaded_config_dict) - logger.debug("configuration loaded_config_dict prepped. Attempting to seralize...") + logger.debug("Configuration loaded_config_dict prepped. Attempting to seralize...") try: loaded_config_dict = ConfigurationSchema().load(data=loaded_config_dict, unknown=marshmallow.EXCLUDE) except marshmallow.ValidationError: logger.exception("Unable to load the configuration.") exit(1) + logger.debug("Seralization successful.") return Config(user=loaded_config_dict, schema=ConfigurationSchema, default=get_default_config()) @@ -704,11 +721,5 @@ def get_config() -> Config: return _load_config() -@functools.lru_cache(None) -def get_default_config() -> BaseConfig: - """Get the default configuration instance of the global Config instance.""" - return BaseConfig() - - config = get_config default = get_default_config diff --git a/tests/modmail/test_config.py b/tests/modmail/test_config.py index a7a580a8..4faad2e9 100644 --- a/tests/modmail/test_config.py +++ b/tests/modmail/test_config.py @@ -19,13 +19,16 @@ def test_config_is_cached(): """Test configuration is cached, helping keep only one version of the configuration in existance.""" for _ in range(2): - assert config.config() == config._CACHED_CONFIG + assert config.config() is config.config() +@pytest.mark.xfail( + reason="Default config is temporarily not cached due to a bug relating to frozen attributes." +) def test_default_config_is_cached(): """Test default configuration is cached, helping keep only one version of the config in existance.""" for _ in range(2): - assert config.default() == config._CACHED_DEFAULT + assert config.default() is config.default() class TestConfigLoaders: From 258fa1b39505aefa8e11e04ea89f16c240114aa5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 12:32:28 -0500 Subject: [PATCH 65/76] fix: use config.user to return configured prefix --- modmail/extensions/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/extensions/meta.py b/modmail/extensions/meta.py index 35139d6f..869a33eb 100644 --- a/modmail/extensions/meta.py +++ b/modmail/extensions/meta.py @@ -43,7 +43,7 @@ async def prefix(self, ctx: commands.Context) -> None: await ctx.send( embed=discord.Embed( title="Current Prefix", - description=f"My currently configured prefix is `{self.bot.config.bot.prefix}`.", + description=f"My currently configured prefix is `{self.bot.config.user.bot.prefix}`.", ) ) From bfe641c45f4645b1d8908086e29d436e2bcefd7b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 16 Nov 2021 12:43:09 -0500 Subject: [PATCH 66/76] chore: export the extended description after the normal description to app.json --- app.json | 2 +- scripts/export_new_config_to_default_config.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app.json b/app.json index e0d64443..5fa7d2cf 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { "env": { "MODMAIL_BOT_TOKEN": { - "description": "Discord bot token. Required to log in to discord.", + "description": "Discord bot token. Required to log in to discord.\nThis is obtainable from https://discord.com/developers/applications", "required": true }, "MODMAIL_BOT_PREFIX": { diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 620b0e4c..854e1f00 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -229,11 +229,13 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada or meta[METADATA_TABLE].export_to_env_template or meta.get("required", False) ): - + description = ( + f"{meta[METADATA_TABLE].description}\n{meta[METADATA_TABLE].extended_description or ''}" + ).strip() options = defaultdict( str, { - "description": meta[METADATA_TABLE].description, + "description": description, "required": meta[METADATA_TABLE].app_json_required or meta.get("required", False), }, ) From fe53cf7b47d13f50df32ae561bf697a96c5d4bf3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 10:29:27 -0500 Subject: [PATCH 67/76] chore: remove operator.attrgetter and implement recursive setters and getters --- modmail/extensions/configuration_manager.py | 37 +++++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index 9c2c0a2e..eee1b932 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -1,6 +1,5 @@ import inspect import logging -import operator import string import types import typing @@ -27,6 +26,27 @@ KeyT = str +def _recursive_getattr(obj: typing.Any, attribute: str) -> typing.Any: + """Get an attribute recursively. All `.` in attribute will be accessed recursively.""" + for name in attribute.split("."): + obj = getattr(obj, name) + return obj + + +def _recursive_setattr(obj: typing.Any, attribute: str, value: typing.Any) -> typing.Any: + """ + Get an attribute recursively. + + All `.` in attribute will be accessed recursively up to the final, which will be set. + """ + if "." in attribute: + root, attr = attribute.rsplit(".", 1) + obj = _recursive_getattr(obj, root) + + setattr(obj, attr, value) + return value + + class UnableToModifyConfig(commands.CommandError): """Raised when a command is unable to modify the configuration.""" @@ -197,18 +217,13 @@ async def set_config_value(self, option: str, new_value: _T) -> typing.Tuple[str Raises an UnableToModifyConfig error if there is an issue """ - if "." in option: - root, name = option.rsplit(".", 1) - else: - root = "" - name = option - if new_value in [marshmallow.missing, attr.NOTHING]: raise UnableToModifyConfig( - f"`{option}` is a required configuration variable and cannot be reset." + f"`{option.rsplit('.', 1)[-1]}` is a required configuration variable and cannot be reset." ) from None + try: - setattr(operator.attrgetter(root)(self.bot.config.user), name, new_value) + _recursive_setattr(self.bot.config.user, option, new_value) except (attr.exceptions.FrozenAttributeError, attr.exceptions.FrozenInstanceError): raise UnableToModifyConfig( f"Unable to set `{option}` as it is frozen and cannot be edited during runtime." @@ -219,7 +234,7 @@ async def set_config_value(self, option: str, new_value: _T) -> typing.Tuple[str @config_group.command(name="set_default", aliases=("set-default",)) async def set_default(self, ctx: Context, option: KeyConverter) -> None: """Reset the provided configuration value to the default.""" - value = operator.attrgetter(option)(self.bot.config.default) + value = _recursive_getattr(self.bot.config.default, option) await self.set_config_value(option, value) await responses.send_positive_response( @@ -258,7 +273,7 @@ async def modify_config_command(self, ctx: Context, option: KeyConverter, value: @config_group.command(name="get", aliases=("show",)) async def get_config(self, ctx: Context, option: KeyConverter) -> None: """Display an existing configuration value.""" - value = operator.attrgetter(option)(self.bot.config.user) + value = _recursive_getattr(self.bot.config.user, option) await responses.send_general_response(ctx, f"value: `{value}`", embed=discord.Embed(title=option)) From 4e7fb22c0bdc79f33b7e52749ea726f60cd935d7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 10:50:24 -0500 Subject: [PATCH 68/76] chore: make the error spam less spammy for missing yaml file and dependency --- modmail/config.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 1050f007..c2a22325 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -610,12 +610,27 @@ def load_yaml(path: os.PathLike) -> dict: else: path = pathlib.Path(path) + # list of tuples containing a message, check, and stop checking. + # this is complicated in order to display the most errors at once to the user. + # this ensures that an attempt to load a yaml file is met with missing + # dependencies and non-existant file at the same time. states = [ - ("The yaml library is not installed.", yaml is not None), - ("The provided yaml config path does not exist.", path.exists()), - ("The provided yaml config file is not a readable file.", path.is_file()), + ("The yaml library is not installed.", yaml is not None, False), + ("The provided yaml config path does not exist.", path.exists(), True), + ("The provided yaml config file is not a regular file.", path.is_file(), False), ] - if errors := "\n".join(msg for msg, check in states if not check): + + errors = list() + for text, check, stop_checking in states: + if check: + continue + errors.append(text) + if stop_checking: + break + + if errors: + # formulate an error message + errors = "Errors occured while attempting to load a yaml configuration:\n" + "\n".join(errors) raise ConfigLoadError(errors) try: From 03fd57fd2a6e2168fbf8cebaa7bc1c1eebed3876 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 11:48:01 -0500 Subject: [PATCH 69/76] tests: fix environment not being replaced soon enough --- tests/modmail/conftest.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/modmail/conftest.py b/tests/modmail/conftest.py index 2e3b5139..ec3baa96 100644 --- a/tests/modmail/conftest.py +++ b/tests/modmail/conftest.py @@ -6,6 +6,9 @@ import pytest +_ORIG_ENVIRON = None + + def pytest_report_header(config) -> str: """Pytest headers.""" return "package: modmail" @@ -24,16 +27,16 @@ def _get_env(): def pytest_configure(): - """Check that the test specific env file exists, and cancel the run if it does not exist.""" + """Load the test specific environment file, exit if it does not exist.""" env = _get_env() if not env.is_file(): pytest.exit(f"Testing specific {env} does not exist. Cancelling test run.", 2) + os.environ.clear() + dotenv.load_dotenv(_get_env(), override=True) -@pytest.fixture(autouse=True, scope="package") -def standardize_environment(): - """Clear environment variables except for the test.env file.""" - env = _get_env() - with unittest.mock.patch.dict(os.environ, clear=True): - dotenv.load_dotenv(env) - yield +def pytest_unconfigure(): + """Reset os.environ to the original environment before the run.""" + if _ORIG_ENVIRON is not None: + os.environ.clear() + os.environ.update(**_ORIG_ENVIRON) From f0ec0674f44886612609812e949ab6b9ebd76401 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 25 Nov 2021 10:56:05 -0500 Subject: [PATCH 70/76] fix: use type() instead of __class__ --- modmail/extensions/configuration_manager.py | 4 ++-- scripts/export_new_config_to_default_config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modmail/extensions/configuration_manager.py b/modmail/extensions/configuration_manager.py index eee1b932..b68d152b 100644 --- a/modmail/extensions/configuration_manager.py +++ b/modmail/extensions/configuration_manager.py @@ -146,7 +146,7 @@ async def convert(self, ctx: Context, arg: str) -> KeyT: # depending on common problems, it is *possible* to add `_` but would require fuzzy matching over # all of the keys since that can also be a valid character name. - fields = get_all_conf_options(config.default().__class__) + fields = get_all_conf_options(type(config.default())) new_arg = "" for c in arg.lower(): @@ -172,7 +172,7 @@ class ConfigurationManager(ModmailCog, name="Configuration Manager"): def __init__(self, bot: ModmailBot): self.bot = bot - self.config_fields = get_all_conf_options(config.default().__class__) + self.config_fields = get_all_conf_options(type(config.default())) @commands.group(name="config", aliases=("cfg", "conf"), invoke_without_command=True) async def config_group(self, ctx: Context) -> None: diff --git a/scripts/export_new_config_to_default_config.py b/scripts/export_new_config_to_default_config.py index 854e1f00..203d8cf6 100644 --- a/scripts/export_new_config_to_default_config.py +++ b/scripts/export_new_config_to_default_config.py @@ -210,7 +210,7 @@ def get_env_vars(klass: type, env_prefix: str = None) -> typing.Dict[str, Metada ENV_EXPORT_FILE.unlink(missing_ok=True) ENV_EXPORT_FILE.touch() - exported = get_env_vars(default.__class__) + exported = get_env_vars(type(default)) app_json_env = dict() From 09a33d4576818b62c4b276092c2fb8e5f91a7048 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 2 Jan 2022 01:02:25 -0500 Subject: [PATCH 71/76] chore: address reviews restructuring, using 'startswith', other updates --- modmail/config.py | 48 ++++++++--------------------------- modmail/{utils => }/errors.py | 6 +++++ modmail/utils/converters.py | 21 +++++++++++++++ modmail/utils/pagination.py | 2 +- 4 files changed, 38 insertions(+), 39 deletions(-) rename modmail/{utils => }/errors.py (56%) create mode 100644 modmail/utils/converters.py diff --git a/modmail/config.py b/modmail/config.py index c2a22325..67491e12 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -16,7 +16,9 @@ import marshmallow.fields import marshmallow.validate +from modmail.errors import ConfigLoadError from modmail.log import ModmailLogger +from modmail.utils.converters import BetterPartialEmojiConverter _log_optional_deps = logging.getLogger("modmail.optional_dependencies") @@ -103,37 +105,11 @@ def _get_config_directory() -> pathlib.Path: ] -class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): - """ - Converts to a :class:`~discord.PartialEmoji`. - - This is done by extracting the animated flag, name and ID from the emoji. - """ - - async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: - """Convert a provided argument into an emoji object.""" - match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) - if match is not None: - groups = match.groupdict() - animated = bool(groups["animated"]) - emoji_id = int(groups["id"]) - name = groups["name"] - return discord.PartialEmoji(name=name, animated=animated, id=emoji_id) - - return discord.PartialEmoji(name=argument) - - def _generate_default_dict() -> defaultdict: """For defaultdicts to default to a defaultdict.""" return defaultdict(_generate_default_dict) -class ConfigLoadError(Exception): - """Exception if the configuration failed to load from a local file.""" - - pass - - class _ColourField(marshmallow.fields.Field): """Class to convert a str or int into a color and deserialize into a string.""" @@ -152,18 +128,16 @@ def convert(self, argument: typing.Union[str, int, discord.Colour]) -> discord.C if not isinstance(argument, str): argument = str(argument) - if argument[0] == "#": + if argument.startswith("#"): return self.parse_hex_number(argument[1:]) - if argument[0:2] == "0x": + if argument.startswith("0x"): rest = argument[2:] # Legacy backwards compatible syntax - if rest.startswith("#"): - return self.parse_hex_number(rest[1:]) return self.parse_hex_number(rest) arg = argument.lower() - if arg[0:3] == "rgb": + if arg.startswith("rgb"): return self.parse_rgb(arg) arg = arg.replace(" ", "_") @@ -374,7 +348,7 @@ class DeveloperConfig: @log_level.validator def _log_level_validator(self, a: attr.Attribute, value: int) -> None: """Validate that log_level is within 0 to 50.""" - if value not in range(0, 50 + 1): + if value not in range(51): raise ValueError("log_level must be an integer within 0 to 50, inclusive.") @@ -556,7 +530,7 @@ def load_env( if not existing_config_dict: existing_config_dict = defaultdict(_generate_default_dict) - new_config_dict = attr.asdict( + return attr.asdict( _build_class( BaseConfig, dotenv_file=env_file, @@ -565,8 +539,6 @@ def load_env( ) ) - return new_config_dict - def load_toml(path: os.PathLike = None) -> defaultdict: """ @@ -577,7 +549,7 @@ def load_toml(path: os.PathLike = None) -> defaultdict: logger.trace(f"function `load_toml` called to load toml file {path}") if path is None: - path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml"),) + path = CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".toml") else: # fully resolve path path = pathlib.Path(path) @@ -606,7 +578,7 @@ def load_yaml(path: os.PathLike) -> dict: """ logger.trace(f"function `load_yaml` called to load yaml file {path}") if path is None: - path = (CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml"),) + path = CONFIG_DIRECTORY / (USER_CONFIG_FILE_NAME + ".yaml") else: path = pathlib.Path(path) @@ -620,7 +592,7 @@ def load_yaml(path: os.PathLike) -> dict: ("The provided yaml config file is not a regular file.", path.is_file(), False), ] - errors = list() + errors = [] for text, check, stop_checking in states: if check: continue diff --git a/modmail/utils/errors.py b/modmail/errors.py similarity index 56% rename from modmail/utils/errors.py rename to modmail/errors.py index 69a2fda7..1f314157 100644 --- a/modmail/utils/errors.py +++ b/modmail/errors.py @@ -8,3 +8,9 @@ class InvalidArgumentError(Exception): """Improper argument.""" pass + + +class ConfigLoadError(Exception): + """Exception if the configuration failed to load from a local file.""" + + pass diff --git a/modmail/utils/converters.py b/modmail/utils/converters.py new file mode 100644 index 00000000..bd748354 --- /dev/null +++ b/modmail/utils/converters.py @@ -0,0 +1,21 @@ +import discord + + +class BetterPartialEmojiConverter(discord.ext.commands.converter.EmojiConverter): + """ + Converts to a :class:`~discord.PartialEmoji`. + + This is done by extracting the animated flag, name and ID from the emoji. + """ + + async def convert(self, _: discord.ext.commands.context.Context, argument: str) -> discord.PartialEmoji: + """Convert a provided argument into an emoji object.""" + match = discord.PartialEmoji._CUSTOM_EMOJI_RE.match(argument) + if match is not None: + groups = match.groupdict() + animated = bool(groups["animated"]) + emoji_id = int(groups["id"]) + name = groups["name"] + return discord.PartialEmoji(name=name, animated=animated, id=emoji_id) + + return discord.PartialEmoji(name=argument) diff --git a/modmail/utils/pagination.py b/modmail/utils/pagination.py index e0f633a4..18ae894f 100644 --- a/modmail/utils/pagination.py +++ b/modmail/utils/pagination.py @@ -13,7 +13,7 @@ from discord.embeds import Embed, EmbedProxy from discord.ext.commands import Paginator as DpyPaginator -from modmail.utils.errors import InvalidArgumentError, MissingAttributeError +from modmail.errors import InvalidArgumentError, MissingAttributeError if TYPE_CHECKING: From 40877671bc789775236180cc5b7d774893847079 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 21:33:07 -0400 Subject: [PATCH 72/76] remove dependency on environs --- .pre-commit-config.yaml | 1 - modmail/__init__.py | 14 +++++++++----- modmail/log.py | 13 ++++++++++++- poetry.lock | 24 +----------------------- pyproject.toml | 1 - requirements.txt | 1 - 6 files changed, 22 insertions(+), 32 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6885c397..4f0f0eb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,6 @@ repos: - coloredlogs - desert - discord.py - - environs - marshmallow - python-dotenv - pyyaml diff --git a/modmail/__init__.py b/modmail/__init__.py index 715badfd..1d3a153b 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -5,13 +5,17 @@ from pathlib import Path import coloredlogs -import environs -from modmail.log import ModmailLogger +from modmail.log import ModmailLogger, get_log_level_from_name -env = environs.Env() -env.read_env(".env", recurse=False) +try: + import dotenv +except ModuleNotFoundError: + pass +else: + dotenv.load_dotenv(".env") + # On windows aiodns's asyncio support relies on APIs like add_reader (which aiodns uses) # are not guaranteed to be available, and in particular are not available when using the # ProactorEventLoop on Windows, this method is only supported with Windows SelectorEventLoop @@ -28,7 +32,7 @@ # this logging level is set to logging.TRACE because if it is not set to the lowest level, # the child level will be limited to the lowest level this is set to. -ROOT_LOG_LEVEL = env.log_level("MODMAIL_LOG_LEVEL", logging.TRACE) +ROOT_LOG_LEVEL = get_log_level_from_name(os.environ.get("MODMAIL_LOG_LEVEL", logging.TRACE)) FMT = "%(asctime)s %(levelname)10s %(name)15s - [%(lineno)5d]: %(message)s" DATEFMT = "%Y/%m/%d %H:%M:%S" diff --git a/modmail/log.py b/modmail/log.py index 65afd23f..85754943 100644 --- a/modmail/log.py +++ b/modmail/log.py @@ -1,5 +1,16 @@ import logging -from typing import Any +from typing import Any, Union + + +def get_log_level_from_name(name: Union[str, int]) -> int: + """Find the logging level given the provided name.""" + if isinstance(name, int): + return name + name = name.upper() + value = getattr(logging, name, "") + if not isinstance(value, int): + raise TypeError("name must be an existing logging level.") + return value class ModmailLogger(logging.Logger): diff --git a/poetry.lock b/poetry.lock index 282af7b3..fb17273e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -311,24 +311,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "environs" -version = "9.3.5" -description = "simplified environment variable parsing" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -marshmallow = ">=3.0.0" -python-dotenv = "*" - -[package.extras] -dev = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url", "flake8 (==4.0.1)", "flake8-bugbear (==21.9.2)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)", "tox"] -django = ["dj-database-url", "dj-email-url", "django-cache-url"] -lint = ["flake8 (==4.0.1)", "flake8-bugbear (==21.9.2)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)"] -tests = ["pytest", "dj-database-url", "dj-email-url", "django-cache-url"] - [[package]] name = "execnet" version = "1.9.0" @@ -1214,7 +1196,7 @@ yaml = ["PyYAML"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "eacd0d76072c6df35c5193d3ad9cb6d99848016175ee452359b6a4402f2f50b9" +content-hash = "0f700351dce1358d3f77c905e29618c3f8416e32cafe6c2c0158e6b5bd6799b9" [metadata.files] aiodns = [ @@ -1507,10 +1489,6 @@ distlib = [ docopt = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, ] -environs = [ - {file = "environs-9.3.5-py2.py3-none-any.whl", hash = "sha256:f5a3ca00afd32a82e098597c7dc8899bea5ed451d31055e518e8c7681cdf1c2b"}, - {file = "environs-9.3.5.tar.gz", hash = "sha256:47128992b854cf7e279e62127100767d765a1ead8d22f355e63c00661550131b"}, -] execnet = [ {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, diff --git a/pyproject.toml b/pyproject.toml index c03392be..d6e2f77d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ coloredlogs = "^15.0" atoml = "^1.0.3" attrs = "^21.2.0" desert = "^2020.11.18" -environs = "~=9.3.3" marshmallow = "~=3.13.0" python-dotenv = "^0.19.0" PyYAML = { version = "^5.4.1", optional = true } diff --git a/requirements.txt b/requirements.txt index 68067e4d..6205a0ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,6 @@ colorama==0.4.4 ; python_version >= "2.7" and python_version != "3.0" and python coloredlogs==15.0.1 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" desert==2020.11.18 ; python_version >= "3.6" discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_full_version >= "3.8.0" -environs==9.3.5 ; python_version >= "3.6" humanfriendly==10.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" idna==3.2 ; python_version >= "3.5" marshmallow-enum==1.5.1 From 5911ff845f316925485c1578d47213e88b42a10b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 21:54:57 -0400 Subject: [PATCH 73/76] fix: pin discord.py --- poetry.lock | 5 ++--- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index fb17273e..ec766640 100644 --- a/poetry.lock +++ b/poetry.lock @@ -293,8 +293,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] [package.source] type = "url" -url = "https://github.com/Rapptz/discord.py/archive/master.zip" - +url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip" [[package]] name = "distlib" version = "0.3.2" @@ -1196,7 +1195,7 @@ yaml = ["PyYAML"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "0f700351dce1358d3f77c905e29618c3f8416e32cafe6c2c0158e6b5bd6799b9" +content-hash = "c06b8fb70a86923177f0c1e3cf3c4be3bea709eac95482a62be9b5e3cf7d4a44" [metadata.files] aiodns = [ diff --git a/pyproject.toml b/pyproject.toml index d6e2f77d..5586ed7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ aiohttp = { extras = ["speedups"], version = "^3.7.4" } arrow = "^1.1.1" colorama = "^0.4.3" coloredlogs = "^15.0" -"discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/master.zip" } +"discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip" } atoml = "^1.0.3" attrs = "^21.2.0" desert = "^2020.11.18" diff --git a/requirements.txt b/requirements.txt index 6205a0ba..fc3854a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ chardet==4.0.0 ; python_version >= "2.7" and python_version != "3.0" and python_ colorama==0.4.4 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" coloredlogs==15.0.1 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" desert==2020.11.18 ; python_version >= "3.6" -discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_full_version >= "3.8.0" +discord.py @ https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip ; python_full_version >= "3.8.0" humanfriendly==10.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" idna==3.2 ; python_version >= "3.5" marshmallow-enum==1.5.1 From 768277e8dd01383bd18cef3a12479ba7fccac384 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 22:01:41 -0400 Subject: [PATCH 74/76] update black --- poetry.lock | 96 ++++++++++++++++---------------------------------- pyproject.toml | 2 +- 2 files changed, 32 insertions(+), 66 deletions(-) diff --git a/poetry.lock b/poetry.lock index ec766640..1119dd5d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -105,29 +105,24 @@ testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3 [[package]] name = "black" -version = "21.9b0" +version = "22.3.0" description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.6.2" [package.dependencies] -click = ">=7.1.2" +click = ">=8.0.0" mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0,<1" +pathspec = ">=0.9.0" platformdirs = ">=2" -regex = ">=2020.1.8" -tomli = ">=0.2.6,<2.0.0" -typing-extensions = [ - {version = ">=3.10.0.0", markers = "python_version < \"3.10\""}, - {version = "!=3.10.0.1", markers = "python_version >= \"3.10\""}, -] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] +d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -python2 = ["typed-ast (>=1.4.2)"] uvloop = ["uvloop (>=0.15.2)"] [[package]] @@ -294,6 +289,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] [package.source] type = "url" url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip" + [[package]] name = "distlib" version = "0.3.2" @@ -1009,14 +1005,6 @@ python-versions = ">=3.6" [package.dependencies] pyyaml = "*" -[[package]] -name = "regex" -version = "2021.8.28" -description = "Alternative regular expression module, to replace re." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "requests" version = "2.26.0" @@ -1195,7 +1183,7 @@ yaml = ["PyYAML"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "c06b8fb70a86923177f0c1e3cf3c4be3bea709eac95482a62be9b5e3cf7d4a44" +content-hash = "4eaebb69142186eb444698004183844720ad7436976eb465ce0b45310f9f4c27" [metadata.files] aiodns = [ @@ -1270,8 +1258,29 @@ attrs = [ {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, ] black = [ - {file = "black-21.9b0-py3-none-any.whl", hash = "sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115"}, - {file = "black-21.9b0.tar.gz", hash = "sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, ] brotlipy = [ {file = "brotlipy-0.7.0-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2"}, @@ -1932,49 +1941,6 @@ pyyaml-env-tag = [ {file = "pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069"}, {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, ] -regex = [ - {file = "regex-2021.8.28-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d05ad5367c90814099000442b2125535e9d77581855b9bee8780f1b41f2b1a2"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3bf1bc02bc421047bfec3343729c4bbbea42605bcfd6d6bfe2c07ade8b12d2a"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f6a808044faae658f546dd5f525e921de9fa409de7a5570865467f03a626fc0"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a617593aeacc7a691cc4af4a4410031654f2909053bd8c8e7db837f179a630eb"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79aef6b5cd41feff359acaf98e040844613ff5298d0d19c455b3d9ae0bc8c35a"}, - {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0fc1f8f06977c2d4f5e3d3f0d4a08089be783973fc6b6e278bde01f0544ff308"}, - {file = "regex-2021.8.28-cp310-cp310-win32.whl", hash = "sha256:6eebf512aa90751d5ef6a7c2ac9d60113f32e86e5687326a50d7686e309f66ed"}, - {file = "regex-2021.8.28-cp310-cp310-win_amd64.whl", hash = "sha256:ac88856a8cbccfc14f1b2d0b829af354cc1743cb375e7f04251ae73b2af6adf8"}, - {file = "regex-2021.8.28-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c206587c83e795d417ed3adc8453a791f6d36b67c81416676cad053b4104152c"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8690ed94481f219a7a967c118abaf71ccc440f69acd583cab721b90eeedb77c"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:328a1fad67445550b982caa2a2a850da5989fd6595e858f02d04636e7f8b0b13"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c7cb4c512d2d3b0870e00fbbac2f291d4b4bf2634d59a31176a87afe2777c6f0"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66256b6391c057305e5ae9209941ef63c33a476b73772ca967d4a2df70520ec1"}, - {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8e44769068d33e0ea6ccdf4b84d80c5afffe5207aa4d1881a629cf0ef3ec398f"}, - {file = "regex-2021.8.28-cp36-cp36m-win32.whl", hash = "sha256:08d74bfaa4c7731b8dac0a992c63673a2782758f7cfad34cf9c1b9184f911354"}, - {file = "regex-2021.8.28-cp36-cp36m-win_amd64.whl", hash = "sha256:abb48494d88e8a82601af905143e0de838c776c1241d92021e9256d5515b3645"}, - {file = "regex-2021.8.28-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b4c220a1fe0d2c622493b0a1fd48f8f991998fb447d3cd368033a4b86cf1127a"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a332404baa6665b54e5d283b4262f41f2103c255897084ec8f5487ce7b9e8e"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c61dcc1cf9fd165127a2853e2c31eb4fb961a4f26b394ac9fe5669c7a6592892"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ee329d0387b5b41a5dddbb6243a21cb7896587a651bebb957e2d2bb8b63c0791"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f60667673ff9c249709160529ab39667d1ae9fd38634e006bec95611f632e759"}, - {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b844fb09bd9936ed158ff9df0ab601e2045b316b17aa8b931857365ea8586906"}, - {file = "regex-2021.8.28-cp37-cp37m-win32.whl", hash = "sha256:4cde065ab33bcaab774d84096fae266d9301d1a2f5519d7bd58fc55274afbf7a"}, - {file = "regex-2021.8.28-cp37-cp37m-win_amd64.whl", hash = "sha256:1413b5022ed6ac0d504ba425ef02549a57d0f4276de58e3ab7e82437892704fc"}, - {file = "regex-2021.8.28-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ed4b50355b066796dacdd1cf538f2ce57275d001838f9b132fab80b75e8c84dd"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28fc475f560d8f67cc8767b94db4c9440210f6958495aeae70fac8faec631797"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc178caebd0f338d57ae445ef8e9b737ddf8fbc3ea187603f65aec5b041248f"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999ad08220467b6ad4bd3dd34e65329dd5d0df9b31e47106105e407954965256"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:808ee5834e06f57978da3e003ad9d6292de69d2bf6263662a1a8ae30788e080b"}, - {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d5111d4c843d80202e62b4fdbb4920db1dcee4f9366d6b03294f45ed7b18b42e"}, - {file = "regex-2021.8.28-cp38-cp38-win32.whl", hash = "sha256:473858730ef6d6ff7f7d5f19452184cd0caa062a20047f6d6f3e135a4648865d"}, - {file = "regex-2021.8.28-cp38-cp38-win_amd64.whl", hash = "sha256:31a99a4796bf5aefc8351e98507b09e1b09115574f7c9dbb9cf2111f7220d2e2"}, - {file = "regex-2021.8.28-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:04f6b9749e335bb0d2f68c707f23bb1773c3fb6ecd10edf0f04df12a8920d468"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b006628fe43aa69259ec04ca258d88ed19b64791693df59c422b607b6ece8bb"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:121f4b3185feaade3f85f70294aef3f777199e9b5c0c0245c774ae884b110a2d"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a577a21de2ef8059b58f79ff76a4da81c45a75fe0bfb09bc8b7bb4293fa18983"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1743345e30917e8c574f273f51679c294effba6ad372db1967852f12c76759d8"}, - {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e1e8406b895aba6caa63d9fd1b6b1700d7e4825f78ccb1e5260551d168db38ed"}, - {file = "regex-2021.8.28-cp39-cp39-win32.whl", hash = "sha256:ed283ab3a01d8b53de3a05bfdf4473ae24e43caee7dcb5584e86f3f3e5ab4374"}, - {file = "regex-2021.8.28-cp39-cp39-win_amd64.whl", hash = "sha256:610b690b406653c84b7cb6091facb3033500ee81089867ee7d59e675f9ca2b73"}, - {file = "regex-2021.8.28.tar.gz", hash = "sha256:f585cbbeecb35f35609edccb95efd95a3e35824cd7752b586503f7e6087303f1"}, -] requests = [ {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, diff --git a/pyproject.toml b/pyproject.toml index 5586ed7e..d7fdc563 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ yaml = ["pyyaml"] pre-commit = "~=2.1" taskipy = "^1.6.0" # linting -black = "^21.7b0" +black = "^22.3.0" flake8 = "~=3.8" flake8-annotations = "~=2.3" flake8-bugbear = "~=20.1" From 62e82fa35c52e52852b43198ed135f77a53ae62b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 22:03:16 -0400 Subject: [PATCH 75/76] update precommit hooks --- .pre-commit-config.yaml | 8 ++++---- modmail/__init__.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f0f0eb4..ba9267d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: - pyyaml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.2.0 hooks: - id: check-case-conflict - id: check-added-large-files @@ -65,19 +65,19 @@ repos: - id: python-use-type-annotations - repo: https://github.com/PyCQA/isort - rev: 5.9.3 + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/asottile/blacken-docs - rev: v1.11.0 + rev: v1.12.1 hooks: - id: blacken-docs additional_dependencies: - black - repo: https://github.com/psf/black - rev: 21.8b0 + rev: 22.3.0 hooks: - id: black language_version: python3 diff --git a/modmail/__init__.py b/modmail/__init__.py index 1d3a153b..708274df 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -28,7 +28,7 @@ logging.addLevelName(logging.NOTICE, "NOTICE") -LOG_FILE_SIZE = 8 * (2 ** 10) ** 2 # 8MB, discord upload limit +LOG_FILE_SIZE = 8 * (2**10) ** 2 # 8MB, discord upload limit # this logging level is set to logging.TRACE because if it is not set to the lowest level, # the child level will be limited to the lowest level this is set to. From 4644d760c5f185ee8899534f9214d91aca6ab365 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 22:18:36 -0400 Subject: [PATCH 76/76] add configuration system to docs --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 11a81a73..ebe927bd 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added Dispatcher system, although it is not hooked into important features like thread creation yet. (#71) - Officially support python 3.10 (#119) - Officially support windows and macos (#121) +- Completely rewrote configuration system (#75) ### Changed