feat/improve configuration

.gitignore

@@ -105,6 +105,7 @@ celerybeat.pid
 
 # Environments
 .env
+!template.env
 !/tests/modmail/test.env
 .venv
 env/
@@ -134,10 +135,11 @@ dmypy.json
 .idea/
 
 # logs
-logs
+/logs/
 
 # Configuration
-*config.toml
+/modmail_config.toml
+/modmail_config.yaml
 
 # Custom docker compose override
 docker-compose.override.yml

.pre-commit-config.yaml

@@ -1,8 +1,30 @@
 ## Pre-commit setup
 
 repos:
+  # its possible to put this at the bottom, but we want the pre-commit-hooks to check these files as well.
+  - repo: local
+    hooks:
+      - id: ensure-default-configuration-is-exported
+        name: Export default configuration
+        language: python
+        entry: poetry run python -m scripts.export_new_config_to_default_config
+        files: '(app\.json|template\.env|modmail\/(config\.py|default_config(\.toml|\.yaml)))$'
+        require_serial: true
+        additional_dependencies:
+          # so apparently these are needed, but the versions don't have to be pinned since it uses the local env
+          # go figure.
+          - atoml
+          - attrs
+          - click
+          - coloredlogs
+          - desert
+          - discord.py
+          - marshmallow
+          - python-dotenv
+          - pyyaml
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
       - id: check-case-conflict
       - id: check-added-large-files
@@ -11,6 +33,7 @@ repos:
       - id: check-yaml
         exclude: 'mkdocs.yml' # Exclude all mkdocs.yml as they use tags i.e. `!!!`
       - id: pretty-format-json
+        exclude: 'app.json'
         args: [--indent=4, --autofix]
       - id: end-of-file-fixer
       - id: no-commit-to-branch
@@ -42,19 +65,19 @@ repos:
       - id: python-use-type-annotations
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
       - id: isort
 
   - repo: https://github.com/asottile/blacken-docs
-    rev: v1.11.0
+    rev: v1.12.1
     hooks:
       - id: blacken-docs
         additional_dependencies:
           - black
 
   - repo: https://github.com/psf/black
-    rev: 21.8b0
+    rev: 22.3.0
     hooks:
       - id: black
         language_version: python3

app.json

@@ -1,8 +1,13 @@
 {
     "env": {
-        "TOKEN": {
-            "description": "Discord bot token. This is from https://discord.com/developers/applications",
+        "MODMAIL_BOT_TOKEN": {
+            "description": "Discord bot token. Required to log in to discord.\nThis is obtainable from https://discord.com/developers/applications",
             "required": true
+        },
+        "MODMAIL_BOT_PREFIX": {
+            "description": "Command prefix.",
+            "required": false,
+            "value": "?"
         }
     },
     "name": "Modmail Bot",

docs/changelog.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added Dispatcher system, although it is not hooked into important features like thread creation yet. (#71)
 - Officially support python 3.10 (#119)
 - Officially support windows and macos (#121)
+- Completely rewrote configuration system (#75)
 
 ### Changed
 

docs/contributing.md

@@ -152,14 +152,14 @@ $ poetry install
 
 ### Set up modmail config
 
-1. Create a copy of `config-default.yml` named `config.yml` in the the `modmail/` directory.
+1. Create a copy of `modmail/default_config.toml` named `modmail_config.toml` in the root of the repository.
 
 === "Linux, macOS"
 
     <div class="termy">
 
     ```console
-    $ cp -v modmail/config-default.toml modmail/config.toml
+    $ cp -v modmail/default_config.toml modmail_config.toml
     ```
 
     </div>
@@ -169,20 +169,37 @@ $ poetry install
     <div class="termy">
 
     ```console
-    $  xcopy /f modmail/config-default.toml modmail/config.toml
+    $  xcopy /f modmail/default_config.toml modmail_config.toml
     ```
 
     </div>
 
+!!! note
+    If you would like, there is optional support for yaml configuration. Make sure that pyyaml is installed with `poetry install --extras yaml`,
+    and copy modmail/defaults_config.yaml to modmail_config.yaml.
+
 2. Set the modmail bot prefix in `bot.prefix`.
 3. In case you are a contributor set `dev.mode.plugin_dev` and `dev.mode.develop` to `true`. The `develop` variable enables the developer bot extensions and `plugin_dev` enables plugin-developer friendly bot extensions.
 4. Create a text file named `.env` in your project root (that's the base folder of your repository):
-    - You can also copy the `.env.template` file to `.env`
+    - You can also copy the `template.env` file to `.env`
 
 !!!note
         The entire file name is literally `.env`
 
-5. Open the file with any text editor and write the bot token to the files in this format: `TOKEN="my_token"`.
+5. Open the file with any text editor and write the bot token to the files in this format: `MODMAIL_BOT_TOKEN="my_token"`.
+
+
+Of the several supported configuration sources, they are loaded in a specific priority. In decreasing priority:
+- `os.environ`
+- .env
+- modmail_config.yaml (if PyYaml is installed and the file exists)
+- modmail_config.toml (if the above file does not exist)
+- defaults
+
+Internally, the actual parsing order may not match the above,
+but the end configuration object will have the above priorty.
+
+
 
 ### Run The Project
 

modmail/__init__.py

@@ -6,9 +6,16 @@
 
 import coloredlogs
 
-from modmail.log import ModmailLogger
+from modmail.log import ModmailLogger, get_log_level_from_name
 
 
+try:
+    import dotenv
+except ModuleNotFoundError:
+    pass
+else:
+    dotenv.load_dotenv(".env")
+
 # On windows aiodns's asyncio support relies on APIs like add_reader (which aiodns uses)
 # are not guaranteed to be available, and in particular are not available when using the
 # ProactorEventLoop on Windows, this method is only supported with Windows SelectorEventLoop
@@ -21,11 +28,11 @@
 logging.addLevelName(logging.NOTICE, "NOTICE")
 
 
-LOG_FILE_SIZE = 8 * (2 ** 10) ** 2  # 8MB, discord upload limit
+LOG_FILE_SIZE = 8 * (2**10) ** 2  # 8MB, discord upload limit
 
 # this logging level is set to logging.TRACE because if it is not set to the lowest level,
 # the child level will be limited to the lowest level this is set to.
-ROOT_LOG_LEVEL = logging.TRACE
+ROOT_LOG_LEVEL = get_log_level_from_name(os.environ.get("MODMAIL_LOG_LEVEL", logging.TRACE))
 FMT = "%(asctime)s %(levelname)10s %(name)15s - [%(lineno)5d]: %(message)s"
 DATEFMT = "%Y/%m/%d %H:%M:%S"
 

modmail/__main__.py

@@ -21,7 +21,7 @@ def main() -> None:
     """Run the bot."""
     patch_embed()
     bot = ModmailBot()
-    bot.run(bot.config.bot.token)
+    bot.run(bot.config.user.bot.token)
 
 
 if __name__ == "__main__":

modmail/bot.py

@@ -11,7 +11,7 @@
 from discord.client import _cleanup_loop
 from discord.ext import commands
 
-from modmail.config import CONFIG
+from modmail.config import config
 from modmail.dispatcher import Dispatcher
 from modmail.log import ModmailLogger
 from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions
@@ -39,7 +39,7 @@ class ModmailBot(commands.Bot):
     dispatcher: Dispatcher
 
     def __init__(self, **kwargs):
-        self.config = CONFIG
+        self.config = config()
         self.start_time: t.Optional[arrow.Arrow] = None  # arrow.utcnow()
         self.http_session: t.Optional[aiohttp.ClientSession] = None
         self.dispatcher = Dispatcher()
@@ -51,7 +51,7 @@ def __init__(self, **kwargs):
         activity = Activity(type=discord.ActivityType.listening, name="users dming me!")
         # listen to messages mentioning the bot or matching the prefix
         # ! NOTE: This needs to use the configuration system to get the prefix from the db once it exists.
-        prefix = commands.when_mentioned_or(CONFIG.bot.prefix)
+        prefix = self.determine_prefix
         # allow only user mentions by default.
         # ! NOTE: This may change in the future to allow roles as well
         allowed_mentions = AllowedMentions(everyone=False, users=True, roles=False, replied_user=True)
@@ -68,6 +68,15 @@ def __init__(self, **kwargs):
             **kwargs,
         )
 
+    @staticmethod
+    async def determine_prefix(bot: "ModmailBot", message: discord.Message) -> t.List[str]:
+        """Dynamically get the updated prefix on every command."""
+        prefixes = []
+        if bot.config.user.bot.prefix_when_mentioned:
+            prefixes.extend(commands.when_mentioned(bot, message))
+        prefixes.append(bot.config.user.bot.prefix)
+        return prefixes
+
     async def create_connectors(self, *args, **kwargs) -> None:
         """Re-create the connector and set up sessions before logging into Discord."""
         # Use asyncio for DNS resolution instead of threads so threads aren't spammed.