Allow specifying which groups under [project.optional-dependencies] a…

…re considered development dependencies (#628)

* add support for pep 621 dev dependencies

---------

Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com>
Co-authored-by: Mathieu Kniewallner <mathieu.kniewallner@gmail.com>

docs/usage.md

@@ -27,9 +27,10 @@ To determine the project's dependencies, _deptry_ will scan the directory it is
     - development dependencies from `[tool.poetry.group.dev.dependencies]` or `[tool.poetry.dev-dependencies]` section
 2. If a `pyproject.toml` file with a `[tool.pdm.dev-dependencies]` section is found, _deptry_ will assume it uses PDM and extract:
     - dependencies from `[project.dependencies]` and `[project.optional-dependencies]` sections
-    - development dependencies from `[tool.pdm.dev-dependencies]` section.
+    - development dependencies from `[tool.pdm.dev-dependencies]` section and from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument.
 3. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract:
-    - dependencies from `[project.dependencies]` and `[project.optional-dependencies]` sections
+    - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`.
+    - development dependencies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument.
 4. If a `requirements.txt` file is found, _deptry_ will extract:
     - dependencies from it
     - development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist
@@ -139,7 +140,7 @@ deptry . --no-ansi
 
 List of patterns to exclude when searching for source files.
 
-- Type: `List[str]`
+- Type: `list[str]`
 - Default: `["venv", "\.venv", "\.direnv", "tests", "\.git", "setup\.py"]`
 - `pyproject.toml` option name: `exclude`
 - CLI option name: `--exclude` (short: `-e`)
@@ -158,7 +159,7 @@ deptry . --exclude "a_directory|a_python_file\.py|a_pattern/.*"
 Additional list of patterns to exclude when searching for source files.
 This extends the patterns set in [Exclude](#exclude), to allow defining patterns while keeping the default list.
 
-- Type: `List[str]`
+- Type: `list[str]`
 - Default: `[]`
 - `pyproject.toml` option name: `extend_exclude`
 - CLI option name: `--extend-exclude` (short: `-ee`)
@@ -176,7 +177,7 @@ deptry . --extend-exclude "a_directory|a_python_file\.py|a_pattern/.*"
 
 A comma-separated list of [rules](rules-violations.md) to ignore.
 
-- Type: `List[str]`
+- Type: `list[str]`
 - Default: `[]`
 - `pyproject.toml` option name: `ignore`
 - CLI option name: `--ignore` (short: `-i`)
@@ -231,7 +232,7 @@ deptry . --ignore-notebooks
 
 List of [`pip` requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) that contain the source dependencies.
 
-- Type: `List[str]`
+- Type: `list[str]`
 - Default: `["requirements.txt"]`
 - `pyproject.toml` option name: `requirements_txt`
 - CLI option name: `--requirements-txt` (short: `-rt`)
@@ -249,7 +250,7 @@ deptry . --requirements-txt requirements.txt,requirements-private.txt
 
 List of [`pip` requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) that contain the source development dependencies.
 
-- Type: `List[str]`
+- Type: `list[str]`
 - Default: `["dev-requirements.txt", "requirements-dev.txt"]`
 - `pyproject.toml` option name: `requirements_txt_dev`
 - CLI option name: `--requirements-txt-dev` (short: `-rtd`)
@@ -267,7 +268,7 @@ deptry . --requirements-txt-dev requirements-dev.txt,requirements-tests.txt
 
 List of Python modules that should be considered as first party ones. This is useful in case _deptry_ is not able to automatically detect modules that should be considered as local ones.
 
-- Type: `List[str]`
+- Type: `list[str]`
 - Default: `[]`
 - `pyproject.toml` option name: `known_first_party`
 - CLI option name: `--known-first-party` (short: `-kf`)
@@ -400,3 +401,44 @@ Multiple package name to module name mappings are joined by a comma (`,`):
 ```shell
 deptry . --package-module-name-map 'foo-python=foo,bar-python=bar'
 ```
+
+#### PEP 621 dev dependency groups
+
+PEP 621 does [not define](https://peps.python.org/pep-0621/#recommend-that-tools-put-development-related-dependencies-into-a-dev-extra) a standard convention for specifying development dependencies. However, deptry offers a mechanism to interpret specific optional dependency groups as development dependencies.
+
+By default, all dependencies under `[project.dependencies]` and `[project.optional-dependencies]` are extracted as regular dependencies. By using the `--pep621-dev-dependency-groups` argument, users can specify which groups defined under `[project.optional-dependencies]` should be treated as development dependencies instead. This is particularly useful for projects that adhere to PEP 621 but do not employ a separate build tool for declaring development dependencies.
+
+For example, consider a project with the following `pyproject.toml`:
+
+```toml
+[project]
+...
+dependencies = [
+    "httpx",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest < 5.0.0",
+]
+plot = [
+    "matplotlib",
+]
+```
+
+By default, `httpx`, `pytest` and `matplotlib` are extracted as regular dependencies. By specifying `--pep621-dev-dependency-groups=test`,
+the dependency `pytest` will be considered a development dependency instead.
+
+- Type: `list[str]`
+- Default: `[]`
+- `pyproject.toml` option name: `pep621_dev_dependency_groups`
+- CLI option name: `--pep621-dev-dependency-groups` (short: `-ddg`)
+- `pyproject.toml` example:
+```toml
+[tool.deptry]
+pep621_dev_dependency_groups = ["test", "docs"]
+```
+- CLI example:
+```shell
+deptry . --pep621-dev-dependency-groups "test,docs"
+```

python/deptry/cli.py

@@ -224,6 +224,17 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b
     default={},
     show_default=False,
 )
+@click.option(
+    "--pep621-dev-dependency-groups",
+    "-ddg",
+    type=COMMA_SEPARATED_TUPLE,
+    help="""For projects that use PEP621 and that do not use a build tool that has its own method of declaring development dependencies,
+    this argument provides the option to specify which groups under [project.optional-dependencies] in pyproject.toml
+    should be considered development dependencies. For example, use `--pep621-dev-dependency-groups tests,docs` to mark the dependencies in
+    the groups 'tests' and 'docs' as development dependencies.""",
+    default=(),
+    show_default=False,
+)
 def deptry(
     root: tuple[Path, ...],
     config: Path,
@@ -238,6 +249,7 @@ def deptry(
     known_first_party: tuple[str, ...],
     json_output: str,
     package_module_name_map: MutableMapping[str, tuple[str, ...]],
+    pep621_dev_dependency_groups: tuple[str, ...],
 ) -> None:
     """Find dependency issues in your Python project.
 
@@ -267,4 +279,5 @@ def deptry(
         known_first_party=known_first_party,
         json_output=json_output,
         package_module_name_map=package_module_name_map,
+        pep621_dev_dependency_groups=pep621_dev_dependency_groups,
     ).run()

python/deptry/core.py

@@ -53,6 +53,7 @@ class Core:
     known_first_party: tuple[str, ...]
     json_output: str
     package_module_name_map: Mapping[str, tuple[str, ...]]
+    pep621_dev_dependency_groups: tuple[str, ...]
 
     def run(self) -> None:
         self._log_config()
@@ -62,6 +63,8 @@ def run(self) -> None:
         ).detect()
         dependencies_extract = self._get_dependencies(dependency_management_format)
 
+        self._log_dependencies(dependencies_extract)
+
         all_python_files = PythonFileFinder(
             self.exclude, self.extend_exclude, self.using_default_exclude, self.ignore_notebooks
         ).get_all_python_files_in(self.root)
@@ -141,9 +144,13 @@ def _get_dependencies(self, dependency_management_format: DependencyManagementFo
         if dependency_management_format is DependencyManagementFormat.POETRY:
             return PoetryDependencyGetter(self.config, self.package_module_name_map).get()
         if dependency_management_format is DependencyManagementFormat.PDM:
-            return PDMDependencyGetter(self.config, self.package_module_name_map).get()
+            return PDMDependencyGetter(
+                self.config, self.package_module_name_map, self.pep621_dev_dependency_groups
+            ).get()
         if dependency_management_format is DependencyManagementFormat.PEP_621:
-            return PEP621DependencyGetter(self.config, self.package_module_name_map).get()
+            return PEP621DependencyGetter(
+                self.config, self.package_module_name_map, self.pep621_dev_dependency_groups
+            ).get()
         if dependency_management_format is DependencyManagementFormat.REQUIREMENTS_TXT:
             return RequirementsTxtDependencyGetter(
                 self.config, self.package_module_name_map, self.requirements_txt, self.requirements_txt_dev
@@ -188,6 +195,20 @@ def _log_config(self) -> None:
             logging.debug("%s: %s", key, value)
         logging.debug("")
 
+    @staticmethod
+    def _log_dependencies(dependencies_extract: DependenciesExtract) -> None:
+        if dependencies_extract.dependencies:
+            logging.debug("The project contains the following dependencies:")
+            for dependency in dependencies_extract.dependencies:
+                logging.debug(dependency)
+            logging.debug("")
+
+        if dependencies_extract.dev_dependencies:
+            logging.debug("The project contains the following dev dependencies:")
+            for dependency in dependencies_extract.dev_dependencies:
+                logging.debug(dependency)
+            logging.debug("")
+
     @staticmethod
     def _exit(violations: list[Violation]) -> None:
         sys.exit(bool(violations))

python/deptry/dependency_getter/base.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
@@ -36,12 +35,3 @@ class DependencyGetter(ABC):
     def get(self) -> DependenciesExtract:
         """Get extracted dependencies and dev dependencies."""
         raise NotImplementedError()
-
-    @staticmethod
-    def _log_dependencies(dependencies: list[Dependency], is_dev: bool = False) -> None:
-        logging.debug("The project contains the following %s:", "dev dependencies" if is_dev else "dependencies")
-
-        for dependency in dependencies:
-            logging.debug(dependency)
-
-        logging.debug("")

python/deptry/dependency_getter/pdm.py

@@ -21,10 +21,10 @@ class PDMDependencyGetter(PEP621DependencyGetter):
     def get(self) -> DependenciesExtract:
         pep_621_dependencies_extract = super().get()
 
-        dev_dependencies = self._get_pdm_dev_dependencies()
-        self._log_dependencies(dev_dependencies, is_dev=True)
-
-        return DependenciesExtract(pep_621_dependencies_extract.dependencies, dev_dependencies)
+        return DependenciesExtract(
+            pep_621_dependencies_extract.dependencies,
+            [*pep_621_dependencies_extract.dev_dependencies, *self._get_pdm_dev_dependencies()],
+        )
 
     def _get_pdm_dev_dependencies(self) -> list[Dependency]:
         """

python/deptry/dependency_getter/pep_621.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import itertools
+import logging
 import re
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
@@ -15,6 +16,7 @@
 
 @dataclass
 class PEP621DependencyGetter(DependencyGetter):
+    pep621_dev_dependency_groups: tuple[str, ...] = ()
     """
     Class to extract dependencies from a pyproject.toml file in which dependencies are specified according to PEP 621. For example:
 
@@ -33,14 +35,25 @@ class PEP621DependencyGetter(DependencyGetter):
             "pytest-cov[all]"
         ]
 
-    Note that both dependencies and optional-dependencies are extracted as regular dependencies. Since PEP-621 does not specify
-    a recommended way to extract development dependencies, we do not attempt to extract any from the pyproject.toml file.
+    Note that by default both dependencies and optional-dependencies are extracted as regular dependencies, since PEP-621 does not specify
+    a recommended way to extract development dependencies. However, if a value is passed for the `pep621_dev_dependency_groups`
+    argument, all dependencies from groups in that argument are considered to be development dependencies. e.g. in the example above, when
+    `pep621_dev_dependency_groups=(test,)`, both `pytest` and `pytest-cov` are returned as development dependencies.
     """
 
     def get(self) -> DependenciesExtract:
-        dependencies = [*self._get_dependencies(), *itertools.chain(*self._get_optional_dependencies().values())]
-        self._log_dependencies(dependencies)
-
+        dependencies = self._get_dependencies()
+        optional_dependencies = self._get_optional_dependencies()
+
+        if self.pep621_dev_dependency_groups:
+            self._check_for_invalid_group_names(optional_dependencies)
+            dev_dependencies, leftover_optional_dependencies = (
+                self._split_development_dependencies_from_optional_dependencies(optional_dependencies)
+            )
+            dependencies = [*dependencies, *leftover_optional_dependencies]
+            return DependenciesExtract(dependencies, dev_dependencies)
+
+        dependencies = [*dependencies, *itertools.chain(*optional_dependencies.values())]
         return DependenciesExtract(dependencies, [])
 
     def _get_dependencies(self) -> list[Dependency]:
@@ -56,6 +69,35 @@ def _get_optional_dependencies(self) -> dict[str, list[Dependency]]:
             for group, dependencies in pyproject_data["project"].get("optional-dependencies", {}).items()
         }
 
+    def _check_for_invalid_group_names(self, optional_dependencies: dict[str, list[Dependency]]) -> None:
+        missing_groups = set(self.pep621_dev_dependency_groups) - set(optional_dependencies.keys())
+        if missing_groups:
+            logging.warning(
+                "Warning: Trying to extract the dependencies from the optional dependency groups %s as development dependencies, "
+                "but the following groups were not found: %s",
+                list(self.pep621_dev_dependency_groups),
+                list(missing_groups),
+            )
+
+    def _split_development_dependencies_from_optional_dependencies(
+        self, optional_dependencies: dict[str, list[Dependency]]
+    ) -> tuple[list[Dependency], list[Dependency]]:
+        """
+        Split the optional dependencies into optional dependencies and development dependencies based on the `pep621_dev_dependency_groups`
+        parameter. Return a tuple with two values: a list of the development dependencies and a list of the remaining 'true' optional dependencies.
+        """
+        dev_dependencies = list(
+            itertools.chain.from_iterable(
+                deps for group, deps in optional_dependencies.items() if group in self.pep621_dev_dependency_groups
+            )
+        )
+        regular_dependencies = list(
+            itertools.chain.from_iterable(
+                deps for group, deps in optional_dependencies.items() if group not in self.pep621_dev_dependency_groups
+            )
+        )
+        return dev_dependencies, regular_dependencies
+
     def _extract_pep_508_dependencies(
         self, dependencies: list[str], package_module_name_map: Mapping[str, Sequence[str]]
     ) -> list[Dependency]:

python/deptry/dependency_getter/poetry.py

@@ -17,13 +17,7 @@ class PoetryDependencyGetter(DependencyGetter):
     """Extract Poetry dependencies from pyproject.toml."""
 
     def get(self) -> DependenciesExtract:
-        dependencies = self._get_poetry_dependencies()
-        self._log_dependencies(dependencies)
-
-        dev_dependencies = self._get_poetry_dev_dependencies()
-        self._log_dependencies(dev_dependencies, is_dev=True)
-
-        return DependenciesExtract(dependencies, dev_dependencies)
+        return DependenciesExtract(self._get_poetry_dependencies(), self._get_poetry_dev_dependencies())
 
     def _get_poetry_dependencies(self) -> list[Dependency]:
         pyproject_data = load_pyproject_toml(self.config)

python/deptry/dependency_getter/requirements_txt.py

@@ -25,7 +25,6 @@ def get(self) -> DependenciesExtract:
                 *(self._get_dependencies_from_requirements_file(file_name) for file_name in self.requirements_txt)
             )
         )
-        self._log_dependencies(dependencies=dependencies)
 
         dev_dependencies = list(
             itertools.chain(
@@ -35,7 +34,6 @@ def get(self) -> DependenciesExtract:
                 )
             )
         )
-        self._log_dependencies(dependencies=dev_dependencies, is_dev=True)
 
         return DependenciesExtract(dependencies, dev_dependencies)
 

tests/data/pep_621_project/pyproject.toml

@@ -19,10 +19,14 @@ dev = [
     "mypy==0.982",
 ]
 test = ["pytest==7.2.0"]
+plot = ["matplotlib"]
 
 [build-system]
 requires = ["setuptools>=61.0.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.deptry]
+pep621_dev_dependency_groups = ["dev"]
+
 [tool.deptry.per_rule_ignores]
 DEP002 = ["pkginfo"]

tests/data/project_with_pdm/pyproject.toml

@@ -28,5 +28,8 @@ test = [
     "pytest-cov>=4.0.0",
 ]
 
+[tool.deptry]
+pep621_dev_dependency_groups = ["bar"]
+
 [tool.deptry.per_rule_ignores]
 DEP002 = ["pkginfo"]

tests/functional/cli/test_cli.py

@@ -418,6 +418,7 @@ def test_cli_verbose(poetry_venv_factory: PoetryVenvFactory) -> None:
 
         assert result.returncode == 1
         assert "The project contains the following dependencies:" in result.stderr
+        assert "The project contains the following dev dependencies:" in result.stderr
         assert get_issues_report(Path(issue_report)) == [
             {
                 "error": {

tests/functional/cli/test_cli_pdm.py

@@ -33,18 +33,6 @@ def test_cli_with_pdm(pdm_venv_factory: PDMVenvFactory) -> None:
                     "column": None,
                 },
             },
-            {
-                "error": {
-                    "code": "DEP002",
-                    "message": "'requests' defined as a dependency but not used in the codebase",
-                },
-                "module": "requests",
-                "location": {
-                    "file": str(Path("pyproject.toml")),
-                    "line": None,
-                    "column": None,
-                },
-            },
             {
                 "error": {
                     "code": "DEP004",