Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: simplify dependency specification detector #654

38 changes: 11 additions & 27 deletions python/deptry/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,8 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING

from deptry.dependency_getter.pdm import PDMDependencyGetter
from deptry.dependency_getter.pep_621 import PEP621DependencyGetter
from deptry.dependency_getter.poetry import PoetryDependencyGetter
from deptry.dependency_getter.requirements_files import RequirementsTxtDependencyGetter
from deptry.dependency_specification_detector import DependencyManagementFormat, DependencySpecificationDetector
from deptry.exceptions import IncorrectDependencyFormatError, UnsupportedPythonVersionError
from deptry.dependency_getter.builder import DependencyGetterBuilder
from deptry.exceptions import UnsupportedPythonVersionError
from deptry.imports.extract import get_imported_modules_from_list_of_files
from deptry.module import ModuleBuilder, ModuleLocations
from deptry.python_file_finder import get_all_python_files_in
Expand Down Expand Up @@ -58,10 +54,15 @@
def run(self) -> None:
self._log_config()

dependency_management_format = DependencySpecificationDetector(
self.config, requirements_files=self.requirements_files
).detect()
dependencies_extract = self._get_dependencies(dependency_management_format)
dependency_getter = DependencyGetterBuilder(

Check warning on line 57 in python/deptry/core.py

View check run for this annotation

Codecov / codecov/patch

python/deptry/core.py#L57

Added line #L57 was not covered by tests
self.config,
self.package_module_name_map,
self.pep621_dev_dependency_groups,
self.requirements_files,
self.requirements_files_dev,
).build()

dependencies_extract = dependency_getter.get()

Check warning on line 65 in python/deptry/core.py

View check run for this annotation

Codecov / codecov/patch

python/deptry/core.py#L65

Added line #L65 was not covered by tests

self._log_dependencies(dependencies_extract)

Expand Down Expand Up @@ -150,23 +151,6 @@
violations, key=operator.attrgetter("location.file", "location.line", "location.column", "error_code")
)

def _get_dependencies(self, dependency_management_format: DependencyManagementFormat) -> DependenciesExtract:
if dependency_management_format is DependencyManagementFormat.POETRY:
return PoetryDependencyGetter(self.config, self.package_module_name_map).get()
if dependency_management_format is DependencyManagementFormat.PDM:
return PDMDependencyGetter(
self.config, self.package_module_name_map, self.pep621_dev_dependency_groups
).get()
if dependency_management_format is DependencyManagementFormat.PEP_621:
return PEP621DependencyGetter(
self.config, self.package_module_name_map, self.pep621_dev_dependency_groups
).get()
if dependency_management_format is DependencyManagementFormat.REQUIREMENTS_FILE:
return RequirementsTxtDependencyGetter(
self.config, self.package_module_name_map, self.requirements_files, self.requirements_files_dev
).get()
raise IncorrectDependencyFormatError

def _get_local_modules(self) -> set[str]:
"""
Get all local Python modules from the source directories and `known_first_party` list.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,44 +1,60 @@
from __future__ import annotations

import logging
from enum import Enum
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Mapping

from deptry.dependency_getter.pdm import PDMDependencyGetter
from deptry.dependency_getter.pep_621 import PEP621DependencyGetter
from deptry.dependency_getter.poetry import PoetryDependencyGetter
from deptry.dependency_getter.requirements_files import RequirementsTxtDependencyGetter
from deptry.exceptions import DependencySpecificationNotFoundError
from deptry.utils import load_pyproject_toml

if TYPE_CHECKING:
from typing import Any

class DependencyManagementFormat(Enum):
PDM = "pdm"
PEP_621 = "pep_621"
POETRY = "poetry"
REQUIREMENTS_FILE = "requirements_files"
from deptry.dependency_getter.base import DependencyGetter


class DependencySpecificationDetector:
@dataclass
class DependencyGetterBuilder:
"""
Class to detect how dependencies are specified:
- Either find a pyproject.toml with a [poetry.tool.dependencies] section
- Otherwise, find a pyproject.toml with a [tool.pdm] section
- Otherwise, find a pyproject.toml with a [project] section
- Otherwise, find a requirements.txt.

"""

def __init__(self, config: Path, requirements_files: tuple[str, ...] = ("requirements.txt",)) -> None:
self.config = config
self.requirements_files = requirements_files
config: Path
package_module_name_map: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
pep621_dev_dependency_groups: tuple[str, ...] = ()
requirements_files: tuple[str, ...] = ()
requirements_files_dev: tuple[str, ...] = ()

def detect(self) -> DependencyManagementFormat:
def build(self) -> DependencyGetter:
pyproject_toml_found = self._project_contains_pyproject_toml()
if pyproject_toml_found and self._project_uses_poetry():
return DependencyManagementFormat.POETRY
if pyproject_toml_found and self._project_uses_pdm():
return DependencyManagementFormat.PDM
if pyproject_toml_found and self._project_uses_pep_621():
return DependencyManagementFormat.PEP_621

if pyproject_toml_found:
pyproject_toml = load_pyproject_toml(self.config)

if self._project_uses_poetry(pyproject_toml):
return PoetryDependencyGetter(self.config, self.package_module_name_map)

if self._project_uses_pdm(pyproject_toml):
return PDMDependencyGetter(self.config, self.package_module_name_map, self.pep621_dev_dependency_groups)

if self._project_uses_pep_621(pyproject_toml):
return PEP621DependencyGetter(
self.config, self.package_module_name_map, self.pep621_dev_dependency_groups
)

if self._project_uses_requirements_files():
return DependencyManagementFormat.REQUIREMENTS_FILE
return RequirementsTxtDependencyGetter(
self.config, self.package_module_name_map, self.requirements_files, self.requirements_files_dev
)

raise DependencySpecificationNotFoundError(self.requirements_files)

Expand All @@ -50,8 +66,8 @@ def _project_contains_pyproject_toml(self) -> bool:
logging.debug("No pyproject.toml found.")
return False

def _project_uses_poetry(self) -> bool:
pyproject_toml = load_pyproject_toml(self.config)
@staticmethod
def _project_uses_poetry(pyproject_toml: dict[str, Any]) -> bool:
try:
pyproject_toml["tool"]["poetry"]["dependencies"]
logging.debug(
Expand All @@ -67,8 +83,8 @@ def _project_uses_poetry(self) -> bool:
else:
return True

def _project_uses_pdm(self) -> bool:
pyproject_toml = load_pyproject_toml(self.config)
@staticmethod
def _project_uses_pdm(pyproject_toml: dict[str, Any]) -> bool:
try:
pyproject_toml["tool"]["pdm"]["dev-dependencies"]
logging.debug(
Expand All @@ -84,22 +100,20 @@ def _project_uses_pdm(self) -> bool:
else:
return True

def _project_uses_pep_621(self) -> bool:
pyproject_toml = load_pyproject_toml(self.config)
try:
pyproject_toml["project"]
@staticmethod
def _project_uses_pep_621(pyproject_toml: dict[str, Any]) -> bool:
if pyproject_toml.get("project"):
logging.debug(
"pyproject.toml contains a [project] section, so PEP 621 is used to specify the project's dependencies."
)
except KeyError:
logging.debug(
"pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's"
" dependencies."
)
return False
else:
return True

logging.debug(
"pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's"
" dependencies."
)
return False

def _project_uses_requirements_files(self) -> bool:
check = any(Path(requirements_files).is_file() for requirements_files in self.requirements_files)
if check:
Expand Down
5 changes: 0 additions & 5 deletions python/deptry/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,6 @@ def __init__(self, requirements_files: tuple[str, ...]) -> None:
)


class IncorrectDependencyFormatError(ValueError):
def __init__(self) -> None:
super().__init__("Incorrect dependency manage format. Only poetry, pdm and requirements.txt are supported.")


class PyprojectFileNotFoundError(FileNotFoundError):
def __init__(self, directory: Path) -> None:
super().__init__(f"No file `pyproject.toml` found in directory {directory}")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
from __future__ import annotations

import logging
import re
from pathlib import Path

import pytest

from deptry.dependency_specification_detector import DependencyManagementFormat, DependencySpecificationDetector
from deptry.dependency_getter.builder import DependencyGetterBuilder
from deptry.dependency_getter.pdm import PDMDependencyGetter
from deptry.dependency_getter.pep_621 import PEP621DependencyGetter
from deptry.dependency_getter.poetry import PoetryDependencyGetter
from deptry.dependency_getter.requirements_files import RequirementsTxtDependencyGetter
from deptry.exceptions import DependencySpecificationNotFoundError
from tests.utils import run_within_dir

Expand All @@ -17,17 +22,8 @@ def test_poetry(tmp_path: Path) -> None:
with pyproject_toml_path.open("w") as f:
f.write('[tool.poetry.dependencies]\nfake = "10"')

spec = DependencySpecificationDetector(pyproject_toml_path).detect()
assert spec == DependencyManagementFormat.POETRY


def test_requirements_files(tmp_path: Path) -> None:
with run_within_dir(tmp_path):
with Path("requirements.txt").open("w") as f:
f.write('foo >= "1.0"')

spec = DependencySpecificationDetector(Path("pyproject.toml")).detect()
assert spec == DependencyManagementFormat.REQUIREMENTS_FILE
spec = DependencyGetterBuilder(pyproject_toml_path).build()
assert isinstance(spec, PoetryDependencyGetter)


def test_pdm_with_dev_dependencies(tmp_path: Path) -> None:
Expand All @@ -40,8 +36,8 @@ def test_pdm_with_dev_dependencies(tmp_path: Path) -> None:
' "scm"}\n[tool.pdm.dev-dependencies]\ngroup=["bar"]'
)

spec = DependencySpecificationDetector(pyproject_toml_path).detect()
assert spec == DependencyManagementFormat.PDM
spec = DependencyGetterBuilder(pyproject_toml_path).build()
assert isinstance(spec, PDMDependencyGetter)


def test_pdm_without_dev_dependencies(tmp_path: Path) -> None:
Expand All @@ -51,8 +47,8 @@ def test_pdm_without_dev_dependencies(tmp_path: Path) -> None:
with pyproject_toml_path.open("w") as f:
f.write('[project]\ndependencies=["foo"]\n[tool.pdm]\nversion = {source = "scm"}')

spec = DependencySpecificationDetector(pyproject_toml_path).detect()
assert spec == DependencyManagementFormat.PEP_621
spec = DependencyGetterBuilder(pyproject_toml_path).build()
assert isinstance(spec, PEP621DependencyGetter)


def test_pep_621(tmp_path: Path) -> None:
Expand All @@ -62,8 +58,8 @@ def test_pep_621(tmp_path: Path) -> None:
with pyproject_toml_path.open("w") as f:
f.write('[project]\ndependencies=["foo"]')

spec = DependencySpecificationDetector(pyproject_toml_path).detect()
assert spec == DependencyManagementFormat.PEP_621
spec = DependencyGetterBuilder(pyproject_toml_path).build()
assert isinstance(spec, PEP621DependencyGetter)


def test_both(tmp_path: Path) -> None:
Expand All @@ -80,17 +76,17 @@ def test_both(tmp_path: Path) -> None:
with Path("requirements.txt").open("w") as f:
f.write('foo >= "1.0"')

spec = DependencySpecificationDetector(pyproject_toml_path).detect()
assert spec == DependencyManagementFormat.POETRY
spec = DependencyGetterBuilder(pyproject_toml_path).build()
assert isinstance(spec, PoetryDependencyGetter)


def test_requirements_files_with_argument(tmp_path: Path) -> None:
def test_requirements_files(tmp_path: Path) -> None:
with run_within_dir(tmp_path):
with Path("req.txt").open("w") as f:
with Path("requirements.txt").open("w") as f:
f.write('foo >= "1.0"')

spec = DependencySpecificationDetector(Path("pyproject.toml"), requirements_files=("req.txt",)).detect()
assert spec == DependencyManagementFormat.REQUIREMENTS_FILE
spec = DependencyGetterBuilder(Path("pyproject.toml"), requirements_files=("requirements.txt",)).build()
assert isinstance(spec, RequirementsTxtDependencyGetter)


def test_requirements_files_with_argument_not_root_directory(tmp_path: Path) -> None:
Expand All @@ -100,16 +96,38 @@ def test_requirements_files_with_argument_not_root_directory(tmp_path: Path) ->
with Path("req/req.txt").open("w") as f:
f.write('foo >= "1.0"')

spec = DependencySpecificationDetector(Path("pyproject.toml"), requirements_files=("req/req.txt",)).detect()
assert spec == DependencyManagementFormat.REQUIREMENTS_FILE
spec = DependencyGetterBuilder(Path("pyproject.toml"), requirements_files=("req/req.txt",)).build()
assert isinstance(spec, RequirementsTxtDependencyGetter)


def test_dependency_specification_not_found_raises_exception(tmp_path: Path) -> None:
with run_within_dir(tmp_path), pytest.raises(
def test_dependency_specification_not_found_raises_exception(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
with run_within_dir(tmp_path):
pyproject_toml_path = Path("pyproject.toml")

with pyproject_toml_path.open("w") as f:
f.write('[build-system]\nrequires = ["maturin>=1.5,<2.0"]\nbuild-backend = "maturin"')

with caplog.at_level(logging.DEBUG), run_within_dir(tmp_path), pytest.raises(
DependencySpecificationNotFoundError,
match=re.escape(
"No file called 'pyproject.toml' with a [tool.poetry.dependencies], [tool.pdm] or [project] section or"
" file(s) called 'req/req.txt' found. Exiting."
),
):
DependencySpecificationDetector(Path("pyproject.toml"), requirements_files=("req/req.txt",)).detect()
DependencyGetterBuilder(Path("pyproject.toml"), requirements_files=("req/req.txt",)).build()

assert caplog.messages == [
"pyproject.toml found!",
(
"pyproject.toml does not contain a [tool.poetry.dependencies] section, so Poetry is not used to specify the"
" project's dependencies."
),
(
"pyproject.toml does not contain a [tool.pdm.dev-dependencies] section, so PDM is not used to specify the"
" project's dependencies."
),
(
"pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's"
" dependencies."
),
]