From 5494d0927f00a483590c88e16def71532741af7a Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Tue, 2 Apr 2024 09:00:58 +0200 Subject: [PATCH] refactor: simplify dependency specification detector (#654) * refactor(spec-detector): dataclass + clearer branches * refactor(dependency-spec): set `requirements_files` default to empty tuple. We already pass the real default value from the CLI, so this avoids duplciating the default. * refactor(dependency-spec): directly return `DependencyGetter` instance * refactor(dependency-spec): rename and move to `dependency_getter` * perf(dependency-getter): load `pyproject.toml` once * refactor(dependency-spec): use `.get()` in `_project_uses_pep_621` * test(dependency-getter): assert debug logs --- python/deptry/core.py | 38 +++------ .../builder.py} | 82 +++++++++++-------- python/deptry/exceptions.py | 5 -- .../test_builder.py} | 76 ++++++++++------- 4 files changed, 106 insertions(+), 95 deletions(-) rename python/deptry/{dependency_specification_detector.py => dependency_getter/builder.py} (56%) rename tests/unit/{test_dependency_specification_detector.py => dependency_getter/test_builder.py} (51%) diff --git a/python/deptry/core.py b/python/deptry/core.py index 2fe63530..04b21db0 100644 --- a/python/deptry/core.py +++ b/python/deptry/core.py @@ -6,12 +6,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from deptry.dependency_getter.pdm import PDMDependencyGetter -from deptry.dependency_getter.pep_621 import PEP621DependencyGetter -from deptry.dependency_getter.poetry import PoetryDependencyGetter -from deptry.dependency_getter.requirements_files import RequirementsTxtDependencyGetter -from deptry.dependency_specification_detector import DependencyManagementFormat, DependencySpecificationDetector -from deptry.exceptions import IncorrectDependencyFormatError, UnsupportedPythonVersionError +from deptry.dependency_getter.builder import DependencyGetterBuilder +from deptry.exceptions import UnsupportedPythonVersionError from deptry.imports.extract import get_imported_modules_from_list_of_files from deptry.module import ModuleBuilder, ModuleLocations from deptry.python_file_finder import get_all_python_files_in @@ -58,10 +54,15 @@ class Core: def run(self) -> None: self._log_config() - dependency_management_format = DependencySpecificationDetector( - self.config, requirements_files=self.requirements_files - ).detect() - dependencies_extract = self._get_dependencies(dependency_management_format) + dependency_getter = DependencyGetterBuilder( + self.config, + self.package_module_name_map, + self.pep621_dev_dependency_groups, + self.requirements_files, + self.requirements_files_dev, + ).build() + + dependencies_extract = dependency_getter.get() self._log_dependencies(dependencies_extract) @@ -150,23 +151,6 @@ def _get_sorted_violations(violations: list[Violation]) -> list[Violation]: violations, key=operator.attrgetter("location.file", "location.line", "location.column", "error_code") ) - def _get_dependencies(self, dependency_management_format: DependencyManagementFormat) -> DependenciesExtract: - if dependency_management_format is DependencyManagementFormat.POETRY: - return PoetryDependencyGetter(self.config, self.package_module_name_map).get() - if dependency_management_format is DependencyManagementFormat.PDM: - return PDMDependencyGetter( - self.config, self.package_module_name_map, self.pep621_dev_dependency_groups - ).get() - if dependency_management_format is DependencyManagementFormat.PEP_621: - return PEP621DependencyGetter( - self.config, self.package_module_name_map, self.pep621_dev_dependency_groups - ).get() - if dependency_management_format is DependencyManagementFormat.REQUIREMENTS_FILE: - return RequirementsTxtDependencyGetter( - self.config, self.package_module_name_map, self.requirements_files, self.requirements_files_dev - ).get() - raise IncorrectDependencyFormatError - def _get_local_modules(self) -> set[str]: """ Get all local Python modules from the source directories and `known_first_party` list. diff --git a/python/deptry/dependency_specification_detector.py b/python/deptry/dependency_getter/builder.py similarity index 56% rename from python/deptry/dependency_specification_detector.py rename to python/deptry/dependency_getter/builder.py index 0cd72d5f..ae64f7dd 100644 --- a/python/deptry/dependency_specification_detector.py +++ b/python/deptry/dependency_getter/builder.py @@ -1,44 +1,60 @@ from __future__ import annotations import logging -from enum import Enum +from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING, Mapping +from deptry.dependency_getter.pdm import PDMDependencyGetter +from deptry.dependency_getter.pep_621 import PEP621DependencyGetter +from deptry.dependency_getter.poetry import PoetryDependencyGetter +from deptry.dependency_getter.requirements_files import RequirementsTxtDependencyGetter from deptry.exceptions import DependencySpecificationNotFoundError from deptry.utils import load_pyproject_toml +if TYPE_CHECKING: + from typing import Any -class DependencyManagementFormat(Enum): - PDM = "pdm" - PEP_621 = "pep_621" - POETRY = "poetry" - REQUIREMENTS_FILE = "requirements_files" + from deptry.dependency_getter.base import DependencyGetter -class DependencySpecificationDetector: +@dataclass +class DependencyGetterBuilder: """ Class to detect how dependencies are specified: - Either find a pyproject.toml with a [poetry.tool.dependencies] section - Otherwise, find a pyproject.toml with a [tool.pdm] section - Otherwise, find a pyproject.toml with a [project] section - Otherwise, find a requirements.txt. - """ - def __init__(self, config: Path, requirements_files: tuple[str, ...] = ("requirements.txt",)) -> None: - self.config = config - self.requirements_files = requirements_files + config: Path + package_module_name_map: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + pep621_dev_dependency_groups: tuple[str, ...] = () + requirements_files: tuple[str, ...] = () + requirements_files_dev: tuple[str, ...] = () - def detect(self) -> DependencyManagementFormat: + def build(self) -> DependencyGetter: pyproject_toml_found = self._project_contains_pyproject_toml() - if pyproject_toml_found and self._project_uses_poetry(): - return DependencyManagementFormat.POETRY - if pyproject_toml_found and self._project_uses_pdm(): - return DependencyManagementFormat.PDM - if pyproject_toml_found and self._project_uses_pep_621(): - return DependencyManagementFormat.PEP_621 + + if pyproject_toml_found: + pyproject_toml = load_pyproject_toml(self.config) + + if self._project_uses_poetry(pyproject_toml): + return PoetryDependencyGetter(self.config, self.package_module_name_map) + + if self._project_uses_pdm(pyproject_toml): + return PDMDependencyGetter(self.config, self.package_module_name_map, self.pep621_dev_dependency_groups) + + if self._project_uses_pep_621(pyproject_toml): + return PEP621DependencyGetter( + self.config, self.package_module_name_map, self.pep621_dev_dependency_groups + ) + if self._project_uses_requirements_files(): - return DependencyManagementFormat.REQUIREMENTS_FILE + return RequirementsTxtDependencyGetter( + self.config, self.package_module_name_map, self.requirements_files, self.requirements_files_dev + ) raise DependencySpecificationNotFoundError(self.requirements_files) @@ -50,8 +66,8 @@ def _project_contains_pyproject_toml(self) -> bool: logging.debug("No pyproject.toml found.") return False - def _project_uses_poetry(self) -> bool: - pyproject_toml = load_pyproject_toml(self.config) + @staticmethod + def _project_uses_poetry(pyproject_toml: dict[str, Any]) -> bool: try: pyproject_toml["tool"]["poetry"]["dependencies"] logging.debug( @@ -67,8 +83,8 @@ def _project_uses_poetry(self) -> bool: else: return True - def _project_uses_pdm(self) -> bool: - pyproject_toml = load_pyproject_toml(self.config) + @staticmethod + def _project_uses_pdm(pyproject_toml: dict[str, Any]) -> bool: try: pyproject_toml["tool"]["pdm"]["dev-dependencies"] logging.debug( @@ -84,22 +100,20 @@ def _project_uses_pdm(self) -> bool: else: return True - def _project_uses_pep_621(self) -> bool: - pyproject_toml = load_pyproject_toml(self.config) - try: - pyproject_toml["project"] + @staticmethod + def _project_uses_pep_621(pyproject_toml: dict[str, Any]) -> bool: + if pyproject_toml.get("project"): logging.debug( "pyproject.toml contains a [project] section, so PEP 621 is used to specify the project's dependencies." ) - except KeyError: - logging.debug( - "pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's" - " dependencies." - ) - return False - else: return True + logging.debug( + "pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's" + " dependencies." + ) + return False + def _project_uses_requirements_files(self) -> bool: check = any(Path(requirements_files).is_file() for requirements_files in self.requirements_files) if check: diff --git a/python/deptry/exceptions.py b/python/deptry/exceptions.py index 2dc2428f..efc8c67a 100644 --- a/python/deptry/exceptions.py +++ b/python/deptry/exceptions.py @@ -16,11 +16,6 @@ def __init__(self, requirements_files: tuple[str, ...]) -> None: ) -class IncorrectDependencyFormatError(ValueError): - def __init__(self) -> None: - super().__init__("Incorrect dependency manage format. Only poetry, pdm and requirements.txt are supported.") - - class PyprojectFileNotFoundError(FileNotFoundError): def __init__(self, directory: Path) -> None: super().__init__(f"No file `pyproject.toml` found in directory {directory}") diff --git a/tests/unit/test_dependency_specification_detector.py b/tests/unit/dependency_getter/test_builder.py similarity index 51% rename from tests/unit/test_dependency_specification_detector.py rename to tests/unit/dependency_getter/test_builder.py index 648f019b..e41a9d2a 100644 --- a/tests/unit/test_dependency_specification_detector.py +++ b/tests/unit/dependency_getter/test_builder.py @@ -1,11 +1,16 @@ from __future__ import annotations +import logging import re from pathlib import Path import pytest -from deptry.dependency_specification_detector import DependencyManagementFormat, DependencySpecificationDetector +from deptry.dependency_getter.builder import DependencyGetterBuilder +from deptry.dependency_getter.pdm import PDMDependencyGetter +from deptry.dependency_getter.pep_621 import PEP621DependencyGetter +from deptry.dependency_getter.poetry import PoetryDependencyGetter +from deptry.dependency_getter.requirements_files import RequirementsTxtDependencyGetter from deptry.exceptions import DependencySpecificationNotFoundError from tests.utils import run_within_dir @@ -17,17 +22,8 @@ def test_poetry(tmp_path: Path) -> None: with pyproject_toml_path.open("w") as f: f.write('[tool.poetry.dependencies]\nfake = "10"') - spec = DependencySpecificationDetector(pyproject_toml_path).detect() - assert spec == DependencyManagementFormat.POETRY - - -def test_requirements_files(tmp_path: Path) -> None: - with run_within_dir(tmp_path): - with Path("requirements.txt").open("w") as f: - f.write('foo >= "1.0"') - - spec = DependencySpecificationDetector(Path("pyproject.toml")).detect() - assert spec == DependencyManagementFormat.REQUIREMENTS_FILE + spec = DependencyGetterBuilder(pyproject_toml_path).build() + assert isinstance(spec, PoetryDependencyGetter) def test_pdm_with_dev_dependencies(tmp_path: Path) -> None: @@ -40,8 +36,8 @@ def test_pdm_with_dev_dependencies(tmp_path: Path) -> None: ' "scm"}\n[tool.pdm.dev-dependencies]\ngroup=["bar"]' ) - spec = DependencySpecificationDetector(pyproject_toml_path).detect() - assert spec == DependencyManagementFormat.PDM + spec = DependencyGetterBuilder(pyproject_toml_path).build() + assert isinstance(spec, PDMDependencyGetter) def test_pdm_without_dev_dependencies(tmp_path: Path) -> None: @@ -51,8 +47,8 @@ def test_pdm_without_dev_dependencies(tmp_path: Path) -> None: with pyproject_toml_path.open("w") as f: f.write('[project]\ndependencies=["foo"]\n[tool.pdm]\nversion = {source = "scm"}') - spec = DependencySpecificationDetector(pyproject_toml_path).detect() - assert spec == DependencyManagementFormat.PEP_621 + spec = DependencyGetterBuilder(pyproject_toml_path).build() + assert isinstance(spec, PEP621DependencyGetter) def test_pep_621(tmp_path: Path) -> None: @@ -62,8 +58,8 @@ def test_pep_621(tmp_path: Path) -> None: with pyproject_toml_path.open("w") as f: f.write('[project]\ndependencies=["foo"]') - spec = DependencySpecificationDetector(pyproject_toml_path).detect() - assert spec == DependencyManagementFormat.PEP_621 + spec = DependencyGetterBuilder(pyproject_toml_path).build() + assert isinstance(spec, PEP621DependencyGetter) def test_both(tmp_path: Path) -> None: @@ -80,17 +76,17 @@ def test_both(tmp_path: Path) -> None: with Path("requirements.txt").open("w") as f: f.write('foo >= "1.0"') - spec = DependencySpecificationDetector(pyproject_toml_path).detect() - assert spec == DependencyManagementFormat.POETRY + spec = DependencyGetterBuilder(pyproject_toml_path).build() + assert isinstance(spec, PoetryDependencyGetter) -def test_requirements_files_with_argument(tmp_path: Path) -> None: +def test_requirements_files(tmp_path: Path) -> None: with run_within_dir(tmp_path): - with Path("req.txt").open("w") as f: + with Path("requirements.txt").open("w") as f: f.write('foo >= "1.0"') - spec = DependencySpecificationDetector(Path("pyproject.toml"), requirements_files=("req.txt",)).detect() - assert spec == DependencyManagementFormat.REQUIREMENTS_FILE + spec = DependencyGetterBuilder(Path("pyproject.toml"), requirements_files=("requirements.txt",)).build() + assert isinstance(spec, RequirementsTxtDependencyGetter) def test_requirements_files_with_argument_not_root_directory(tmp_path: Path) -> None: @@ -100,16 +96,38 @@ def test_requirements_files_with_argument_not_root_directory(tmp_path: Path) -> with Path("req/req.txt").open("w") as f: f.write('foo >= "1.0"') - spec = DependencySpecificationDetector(Path("pyproject.toml"), requirements_files=("req/req.txt",)).detect() - assert spec == DependencyManagementFormat.REQUIREMENTS_FILE + spec = DependencyGetterBuilder(Path("pyproject.toml"), requirements_files=("req/req.txt",)).build() + assert isinstance(spec, RequirementsTxtDependencyGetter) -def test_dependency_specification_not_found_raises_exception(tmp_path: Path) -> None: - with run_within_dir(tmp_path), pytest.raises( +def test_dependency_specification_not_found_raises_exception(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + with run_within_dir(tmp_path): + pyproject_toml_path = Path("pyproject.toml") + + with pyproject_toml_path.open("w") as f: + f.write('[build-system]\nrequires = ["maturin>=1.5,<2.0"]\nbuild-backend = "maturin"') + + with caplog.at_level(logging.DEBUG), run_within_dir(tmp_path), pytest.raises( DependencySpecificationNotFoundError, match=re.escape( "No file called 'pyproject.toml' with a [tool.poetry.dependencies], [tool.pdm] or [project] section or" " file(s) called 'req/req.txt' found. Exiting." ), ): - DependencySpecificationDetector(Path("pyproject.toml"), requirements_files=("req/req.txt",)).detect() + DependencyGetterBuilder(Path("pyproject.toml"), requirements_files=("req/req.txt",)).build() + + assert caplog.messages == [ + "pyproject.toml found!", + ( + "pyproject.toml does not contain a [tool.poetry.dependencies] section, so Poetry is not used to specify the" + " project's dependencies." + ), + ( + "pyproject.toml does not contain a [tool.pdm.dev-dependencies] section, so PDM is not used to specify the" + " project's dependencies." + ), + ( + "pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's" + " dependencies." + ), + ]