Add linting of code examples in docstrings

.github/ISSUE_TEMPLATE/bug.yml

@@ -61,6 +61,7 @@ body:
         - Tools to configure nox (part:nox)
         - Tools to configure protobufs (part:protobuf)
         - Tools to configure the CI (part:ci)
+        - Tools to configure pytest (part:pytest)
     validations:
       required: true
   - type: textarea

.github/keylabeler.yml

@@ -27,3 +27,4 @@ labelMappings:
   "part:model-only": "part:model-only"
   "part:nox": "part:nox"
   "part:protobuf": "part:protobuf"
+  "part:pytest": "part:pytest"

.github/labeler.yml

@@ -69,3 +69,9 @@
       - "src/frequenz/repo/config/protobuf*"
     all:
       - "!tests*/**"
+
+"part:pytest":
+  - any:
+      - "src/frequenz/repo/config/pytest"
+    all:
+      - "!tests*/**"

MANIFEST.in

@@ -6,6 +6,7 @@ exclude CODEOWNERS
 exclude CONTRIBUTING.md
 exclude mkdocs.yml
 exclude noxfile.py
+exclude src/conftest.py
 recursive-exclude .github *
 recursive-exclude cookiecutter *
 recursive-exclude docs *

README.md

@@ -42,7 +42,7 @@ directory will be created with the generated project name. For example:
 cd ~/devel
 cookiecutter gh:frequenz-floss/frequenz-repo-config-python \
     --directory=cookiecutter \
-    --checkout v0.4.0
+    --checkout v0.5.0
 ```
 
 This command will prompt you for the project type, name, and other

RELEASE_NOTES.md

@@ -2,7 +2,7 @@
 
 ## Summary
 
-<!-- Here goes a general summary of what this release is about -->
+This release adds linting of code examples in *docstrings*, a workflow to check if PRs have updated the release notes and an [editorconfig](https://editorconfig.org/) file, as well as a bunch of bug fixes.
 
 ## Upgrading
 
@@ -38,6 +38,12 @@
 
 ## New Features
 
+- Add support for linting code examples found in *docstrings*.
+
+  A new module `frequenz.repo.config.pytest.examples` is added with an utility function to be able to easily collect and lint code examples in *docstrings*.
+
+  There is also a new optional dependency `extra-lint-examples` to easily pull the dependencies needed to do this linting. Please have a look at the documentation in the `frequenz.repo.config` package for more details.
+
 ### Cookiecutter template
 
 - Add a new GitHub workflow to check that release notes were updated.
@@ -52,6 +58,14 @@
 
 - Add an `.editorconfig` file to ensure a common basic editor configuration for different file types.
 
+- Add a `pytest` hook to collect and lint code examples found in *docstrings* using `pylint`.
+
+  Examples found in code *docstrings* in the `src/` directory will now be collected and checked using `pylint`. This is done via the file `src/conftest.py`, which hooks into `pytest`, so to only check the examples you can run `pylint src`.
+
+  !!! info
+
+      There is a bug in the library used to extract the examples that prevents from collecting examples from `__init__.py` files. See https://github.com/frequenz-floss/frequenz-repo-config-python/issues/113 for more details.
+
 ## Bug Fixes
 
 - The distribution package doesn't include tests and other useless files anymore.

cookiecutter/{{cookiecutter.github_repo_name}}/MANIFEST.in

@@ -9,6 +9,7 @@ exclude CODEOWNERS
 exclude CONTRIBUTING.md
 exclude mkdocs.yml
 exclude noxfile.py
+exclude src/conftest.py
 recursive-exclude .github *
 recursive-exclude docs *
 {%- if cookiecutter.type == "api" %}

cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml

@@ -5,7 +5,7 @@
 requires = [
   "setuptools == 67.7.2",
   "setuptools_scm[toml] == 7.1.0",
-  "frequenz-repo-config[{{cookiecutter.type}}] == 0.4.0",
+  "frequenz-repo-config[{{cookiecutter.type}}] == 0.5.0",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -70,7 +70,7 @@ dev-mkdocs = [
   "mkdocs-material == 9.1.16",
   "mkdocs-section-index == 0.3.5",
   "mkdocstrings[python] == 0.22.0",
-  "frequenz-repo-config[{{cookiecutter.type}}] == 0.4.0",
+  "frequenz-repo-config[{{cookiecutter.type}}] == 0.5.0",
 ]
 dev-mypy = [
   "mypy == 1.2.0",
@@ -79,23 +79,22 @@ dev-mypy = [
 ]
 dev-noxfile = [
   "nox == 2023.4.22",
-  "frequenz-repo-config[{{cookiecutter.type}}] == 0.4.0",
+  "frequenz-repo-config[{{cookiecutter.type}}] == 0.5.0",
 ]
 dev-pylint = [
   "pylint == 2.17.3",
   # For checking the noxfile, docs/ script, and tests
   "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-noxfile,dev-pytest]",
 ]
-{%- if cookiecutter.type == "api" %}
-dev-pytest = ["pytest == 7.3.1"]
-{%- else %}
 dev-pytest = [
   "pytest == 7.3.1",
+  "frequenz-repo-config[extra-lint-examples] == 0.5.0",
+{%- if cookiecutter.type != "api" %}
   "pytest-mock == 3.10.0",
   "pytest-asyncio == 0.21.0",
   "async-solipsism == 0.5",
-]
 {%- endif %}
+]
 dev = [
   "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-docstrings,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
 ]
@@ -138,7 +137,7 @@ disable = [
 
 [tool.pytest.ini_options]
 {%- if cookiecutter.type != "api" %}
-testpaths = ["tests"]
+testpaths = ["tests", "src"]
 asyncio_mode = "auto"
 required_plugins = ["pytest-asyncio", "pytest-mock"]
 {%- else %}
@@ -147,7 +146,7 @@ testpaths = ["pytests"]
 {%- if cookiecutter.type != "api" %}
 
 [[tool.mypy.overrides]]
-module = ["async_solipsism", "async_solipsism.*"]
+module = ["async_solipsism", "async_solipsism.*", "sybil", "sybil.*"]
 ignore_missing_imports = true
 {%- endif %}
 {%- if cookiecutter.type == "api" %}

cookiecutter/{{cookiecutter.github_repo_name}}/src/conftest.py

@@ -0,0 +1,13 @@
+# License: {{cookiecutter.license}}
+# Copyright © {% now 'utc', '%Y' %} {{cookiecutter.author_name}}
+
+"""Validate docstring code examples.
+
+Code examples are often wrapped in triple backticks (```) within docstrings.
+This plugin extracts these code examples and validates them using pylint.
+"""
+
+from frequenz.repo.config.pytest import examples
+from sybil import Sybil
+
+pytest_collect_file = Sybil(**examples.get_sybil_arguments()).pytest()

docs/index.md

@@ -35,7 +35,7 @@ Then simply run [Cookiecutter] where you want to create the new project:
 
 ```sh
 cookiecutter gh:frequenz-floss/frequenz-repo-config-python \
-    --directory=cookiecutter --checkout v0.4.0
+    --directory=cookiecutter --checkout v0.5.0
 ```
 
 This command will prompt you for the project type, name, and other

mkdocs.yml

@@ -109,6 +109,7 @@ plugins:
             - https://nox.thea.codes/en/stable/objects.inv
             - https://oprypin.github.io/mkdocs-gen-files/objects.inv
             - https://setuptools.pypa.io/en/latest/objects.inv
+            - https://sybil.readthedocs.io/en/stable/objects.inv
             - https://typing-extensions.readthedocs.io/en/stable/objects.inv
   - search
   - section-index

pyproject.toml

@@ -56,6 +56,11 @@ api = [
 app = []
 lib = []
 model = []
+extra-lint-examples = [
+  "pylint >= 2.17.3, < 3",
+  "pytest >= 7.3.0, < 8",
+  "sybil >= 5.0.3, < 6",
+]
 dev-docstrings = [
   "pydocstyle == 6.3.0",
   "darglint == 1.8.1",
@@ -78,13 +83,15 @@ dev-mypy = [
 ]
 dev-noxfile = ["nox == 2023.4.22"]
 dev-pylint = [
-  "pylint == 2.17.3",
+  # dev-pytest already defines a dependency to pylint because of the examples
   # For checking the noxfile, docs/ script, and tests
   "frequenz-repo-config[dev-mkdocs,dev-noxfile,dev-pytest]",
 ]
 dev-pytest = [
-  "pytest == 7.3.1",
+  "pytest == 7.4.0",
+  "pylint == 2.17.3",      # We need this to check for the examples
   "cookiecutter == 2.1.1", # For checking the cookiecutter scripts
+  "sybil == 5.0.3",        # Should be consistent with the extra-lint-examples dependency
 ]
 dev = [
   "frequenz-repo-config[dev-mkdocs,dev-docstrings,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
@@ -124,11 +131,11 @@ disable = [
 ]
 
 [[tool.mypy.overrides]]
-module = ["cookiecutter", "cookiecutter.*"]
+module = ["cookiecutter", "cookiecutter.*", "sybil", "sybil.*"]
 ignore_missing_imports = true
 
 [tool.pytest.ini_options]
-testpaths = ["tests"]
+testpaths = ["src", "tests"]
 markers = [
   "integration: integration tests (deselect with '-m \"not integration\"')",
 ]

src/conftest.py

@@ -0,0 +1,14 @@
+# License: MIT
+# Copyright © 2023 Frequenz Energy-as-a-Service GmbH
+
+"""Validate docstring code examples.
+
+Code examples are often wrapped in triple backticks (```) within docstrings.
+This plugin extracts these code examples and validates them using pylint.
+"""
+
+from sybil import Sybil
+
+from frequenz.repo.config.pytest import examples
+
+pytest_collect_file = Sybil(**examples.get_sybil_arguments()).pytest()

src/frequenz/repo/config/__init__.py

@@ -207,6 +207,77 @@
         - path/to/my/custom/script.py
 ```
 
+## `pytest` (running tests)
+
+### Linting examples in the source code's *docstrings*
+
+To make sure the examples included in your source code's *docstrings* are valid, you can
+use [`pytest`](https://pypi.org/project/pytest/) to automatically collect all the
+examples wrapped in triple backticks (````python`) within our docstrings and validate
+them using [`pylint`](https://pypi.org/project/pylint/).
+
+To do so there is some setup that's needed:
+
+1. Add a `conftest.py` file to the root directory containing your source code with the
+   following contents:
+
+    ```python
+    from frequenz.repo.config.pytest import examples
+    from sybil import Sybil
+
+    pytest_collect_file = Sybil(**examples.get_sybil_arguments()).pytest()
+    ```
+
+    Unfortunately, because of how Sybil works, the [`Sybil`][sybil.Sybil] class needs to
+    be instantiated in the `conftest.py` file. To easily do this, the convenience
+    function
+    [`get_sybil_arguments()`][frequenz.repo.config.pytest.examples.get_sybil_arguments]
+    is provided to get the arguments to pass to the `Sybil()` constructor to be able to
+    collect and lint the examples.
+
+2. Add the following configuration to your `pyproject.toml` file (see
+   the [`nox` section](#pyprojecttoml-configuration) for details on how to configure
+   dependencies to play nicely with `nox`):
+
+    ```toml
+    [project.optional-dependencies]
+    # ...
+    dev-pytest = [
+        # ...
+        "frequenz-repo-config[extra-lint-examples] == 0.5.0",
+    ]
+    # ...
+    [[tool.mypy.overrides]]
+    module = [
+        # ...
+        "sybil",
+        "sybil.*",
+    ]
+    ignore_missing_imports = true
+    # ...
+    [tool.pytest.ini_options]
+    testpaths = [
+        # ...
+        "src",
+    ]
+    ```
+
+   This will make sure that you have the appropriate dependencies installed to run the
+   the tests linting and that `mypy` doesn't complain about the `sybil` module not being
+   typed.
+
+3. Exclude the `src/conftest.py` file from the distribution package, as it shouldn't be
+   shipped with the code, it is only for delelopment purposes. To do so, add the
+   following line to the `MANIFEST.in` file:
+
+    ```
+    # ...
+    exclude src/conftest.py
+    ```
+
+Now you should be able to run `nox -s pytest` (or `pytest` directly) and see the tests
+linting the examples in your code's *docstrings*.
+
 # APIs
 
 ## Protobuf configuation
@@ -286,7 +357,7 @@
 requires = [
   "setuptools >= 67.3.2, < 68",
   "setuptools_scm[toml] >= 7.1.0, < 8",
-  "frequenz-repo-config[api] >= 0.3.0, < 0.4.0",
+  "frequenz-repo-config[api] >= 0.5.0, < 0.6.0",
 ]
 build-backend = "setuptools.build_meta"
 

src/frequenz/repo/config/mkdocs.py

@@ -35,7 +35,9 @@ def _is_internal(path_parts: Tuple[str, ...]) -> bool:
     def with_underscore_not_init(part: str) -> bool:
         return part.startswith("_") and part != "__init__"
 
-    return any(p for p in path_parts if with_underscore_not_init(p))
+    is_conftest = len(path_parts) == 1 and path_parts[0] == "conftest"
+
+    return is_conftest or any(p for p in path_parts if with_underscore_not_init(p))
 
 
 def generate_python_api_pages(

src/frequenz/repo/config/pytest/__init__.py

@@ -0,0 +1,12 @@
+# License: MIT
+# Copyright © 2023 Frequenz Energy-as-a-Service GmbH
+
+"""Pytest utilities.
+
+This package contains utilities for testing with [`pytest`](https://pypi.org/project/pytest/).
+
+The following modules are available:
+
+- [`examples`][frequenz.repo.config.pytest.examples]: Utilities to enable linting of
+  code examples in docstrings.
+"""