From d5636feae9651954bab7175d19e1f79bb8d298f9 Mon Sep 17 00:00:00 2001 From: David Vujic Date: Mon, 15 Jan 2024 16:39:19 +0100 Subject: [PATCH 1/5] fix(cli): poly check also for package sub-dependencies (package, not the dist name) --- components/polylith/commands/check.py | 2 +- components/polylith/distributions/core.py | 36 +++++++++++++++++-- .../polylith/distributions/test_core.py | 15 +++++++- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/components/polylith/commands/check.py b/components/polylith/commands/check.py index 57fbd5ea..a546ee4d 100644 --- a/components/polylith/commands/check.py +++ b/components/polylith/commands/check.py @@ -14,7 +14,7 @@ def run(root: Path, ns: str, project_data: dict, options: dict) -> bool: name = project_data["name"] collected_imports = check.report.collect_all_imports(root, ns, project_data) - dists = importlib.metadata.distributions() + dists = list(importlib.metadata.distributions()) known_aliases = distributions.distributions_packages(dists) known_aliases.update(alias.parse(library_alias)) diff --git a/components/polylith/distributions/core.py b/components/polylith/distributions/core.py index 809b8fa0..e6de5e4c 100644 --- a/components/polylith/distributions/core.py +++ b/components/polylith/distributions/core.py @@ -1,6 +1,10 @@ import importlib.metadata +import re from functools import reduce -from typing import Dict, List +from typing import Dict, List, Set + + +SUB_DEP_SEPARATORS = r"[\s!=;><\^~]" def top_level_packages(dist) -> List[str]: @@ -20,9 +24,35 @@ def map_packages(acc, dist) -> dict: return {**acc, **mapped_packages(dist)} +def only_package_name(package: str) -> str: + parts = re.split(SUB_DEP_SEPARATORS, package) + + return str(parts[0]) + + +def dist_subpackages(dist) -> Set[str]: + packages = importlib.metadata.requires(dist.metadata["name"]) or [] + + return {only_package_name(p) for p in packages} + + +def distributions_subpackages(dists) -> Set[str]: + res = [dist_subpackages(dist) for dist in dists] + + return set().union(*res) + + def distributions_packages(dists) -> Dict[str, List[str]]: - """Return a mapping of top-level packages to their distributions.""" - return reduce(map_packages, dists, {}) + """Return a mapping of top-level packages to their distributions. + + Additional dist sub-dependency package names are appended to the result. + """ + mapped: dict = reduce(map_packages, dists, {}) + + sub_packages = distributions_subpackages(dists) + reshaped = {s: [s] for s in sub_packages} + + return {**reshaped, **mapped} def get_distributions(project_dependencies: set) -> list: diff --git a/test/components/polylith/distributions/test_core.py b/test/components/polylith/distributions/test_core.py index 5d866063..e4595b30 100644 --- a/test/components/polylith/distributions/test_core.py +++ b/test/components/polylith/distributions/test_core.py @@ -4,7 +4,7 @@ def test_distribution_packages(): - dists = importlib.metadata.distributions() + dists = list(importlib.metadata.distributions()) res = distributions.distributions_packages(dists) @@ -13,3 +13,16 @@ def test_distribution_packages(): assert res.get(expected_dist) is not None assert res[expected_dist] == [expected_package] + + +def test_parse_package_name_from_dist_requires(): + assert "greenlet" == distributions.core.only_package_name("greenlet !=0.4.17") + assert "mysqlclient" == distributions.core.only_package_name( + "mysqlclient >=1.4.0 ; extra == 'mysql'" + ) + assert "typing-extensions" == distributions.core.only_package_name( + "typing-extensions>=4.6.0)" + ) + assert "pymysql" == distributions.core.only_package_name( + "pymysql ; extra == 'pymysql'" + ) From a4761dcd999ba8a57e36462c1b7aa35087cb30b4 Mon Sep 17 00:00:00 2001 From: David Vujic Date: Mon, 15 Jan 2024 16:39:58 +0100 Subject: [PATCH 2/5] fix(cli): poly check also for package sub-dependencies (package, not the dist name) --- components/polylith/distributions/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/polylith/distributions/core.py b/components/polylith/distributions/core.py index e6de5e4c..353d67dd 100644 --- a/components/polylith/distributions/core.py +++ b/components/polylith/distributions/core.py @@ -45,7 +45,7 @@ def distributions_subpackages(dists) -> Set[str]: def distributions_packages(dists) -> Dict[str, List[str]]: """Return a mapping of top-level packages to their distributions. - Additional dist sub-dependency package names are appended to the result. + Additional dist sub-dependency package names (without dist names) are appended to the result. """ mapped: dict = reduce(map_packages, dists, {}) From c56846c2879c2830e53b6a39f2d8facdd9f215bd Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 16 Jan 2024 09:01:29 +0100 Subject: [PATCH 3/5] wip(cli): gather package sub-dependencies when running poly check --- components/polylith/commands/check.py | 7 +++ components/polylith/distributions/__init__.py | 8 ++- components/polylith/distributions/core.py | 50 +++++++++---------- .../polylith/distributions/test_core.py | 34 +++++++++---- 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/components/polylith/commands/check.py b/components/polylith/commands/check.py index a546ee4d..267b2aea 100644 --- a/components/polylith/commands/check.py +++ b/components/polylith/commands/check.py @@ -19,6 +19,13 @@ def run(root: Path, ns: str, project_data: dict, options: dict) -> bool: known_aliases = distributions.distributions_packages(dists) known_aliases.update(alias.parse(library_alias)) + """ + WIP: + dist_packages = distributions.distributions_packages(dists) + sub_packages = distributions.distributions_sub_packages(dists) + custom_aliases = alias.parse(library_alias) + """ + extra = alias.pick(known_aliases, third_party_libs) libs = third_party_libs.union(extra) diff --git a/components/polylith/distributions/__init__.py b/components/polylith/distributions/__init__.py index d9951bf7..959031e4 100644 --- a/components/polylith/distributions/__init__.py +++ b/components/polylith/distributions/__init__.py @@ -1,3 +1,7 @@ -from polylith.distributions.core import distributions_packages, get_distributions +from polylith.distributions.core import ( + distributions_packages, + distributions_sub_packages, + get_distributions, +) -__all__ = ["distributions_packages", "get_distributions"] +__all__ = ["distributions_packages", "distributions_sub_packages", "get_distributions"] diff --git a/components/polylith/distributions/core.py b/components/polylith/distributions/core.py index 353d67dd..124c5ff9 100644 --- a/components/polylith/distributions/core.py +++ b/components/polylith/distributions/core.py @@ -1,58 +1,56 @@ import importlib.metadata import re from functools import reduce -from typing import Dict, List, Set +from typing import Dict, List SUB_DEP_SEPARATORS = r"[\s!=;><\^~]" -def top_level_packages(dist) -> List[str]: - top_level = dist.read_text("top_level.txt") +def parse_sub_package_name(dependency: str) -> str: + parts = re.split(SUB_DEP_SEPARATORS, dependency) - return str.split(top_level or "") + return str(parts[0]) -def mapped_packages(dist) -> dict: - packages = top_level_packages(dist) +def dist_subpackages(dist) -> dict: name = dist.metadata["name"] + dependencies = importlib.metadata.requires(name) or [] - return {name: packages} if packages else {} + parsed_package_names = {parse_sub_package_name(d) for d in dependencies} + return {name: parsed_package_names} -def map_packages(acc, dist) -> dict: - return {**acc, **mapped_packages(dist)} +def map_sub_packages(acc, dist) -> dict: + return {**acc, **dist_subpackages(dist)} -def only_package_name(package: str) -> str: - parts = re.split(SUB_DEP_SEPARATORS, package) - return str(parts[0]) +def top_level_packages(dist) -> List[str]: + top_level = dist.read_text("top_level.txt") + return str.split(top_level or "") -def dist_subpackages(dist) -> Set[str]: - packages = importlib.metadata.requires(dist.metadata["name"]) or [] - return {only_package_name(p) for p in packages} +def mapped_packages(dist) -> dict: + packages = top_level_packages(dist) + name = dist.metadata["name"] + return {name: packages} if packages else {} -def distributions_subpackages(dists) -> Set[str]: - res = [dist_subpackages(dist) for dist in dists] - return set().union(*res) +def map_packages(acc, dist) -> dict: + return {**acc, **mapped_packages(dist)} def distributions_packages(dists) -> Dict[str, List[str]]: - """Return a mapping of top-level packages to their distributions. - - Additional dist sub-dependency package names (without dist names) are appended to the result. - """ - mapped: dict = reduce(map_packages, dists, {}) + """Return a mapping of top-level packages to their distributions.""" + return reduce(map_packages, dists, {}) - sub_packages = distributions_subpackages(dists) - reshaped = {s: [s] for s in sub_packages} - return {**reshaped, **mapped} +def distributions_sub_packages(dists) -> Dict[str, List[str]]: + """Return the dependencies of each distribution.""" + return reduce(map_sub_packages, dists, {}) def get_distributions(project_dependencies: set) -> list: diff --git a/test/components/polylith/distributions/test_core.py b/test/components/polylith/distributions/test_core.py index e4595b30..fb02b896 100644 --- a/test/components/polylith/distributions/test_core.py +++ b/test/components/polylith/distributions/test_core.py @@ -16,13 +16,27 @@ def test_distribution_packages(): def test_parse_package_name_from_dist_requires(): - assert "greenlet" == distributions.core.only_package_name("greenlet !=0.4.17") - assert "mysqlclient" == distributions.core.only_package_name( - "mysqlclient >=1.4.0 ; extra == 'mysql'" - ) - assert "typing-extensions" == distributions.core.only_package_name( - "typing-extensions>=4.6.0)" - ) - assert "pymysql" == distributions.core.only_package_name( - "pymysql ; extra == 'pymysql'" - ) + expected = { + "greenlet": "greenlet !=0.4.17", + "mysqlclient": "mysqlclient >=1.4.0 ; extra == 'mysql'", + "typing-extensions": "typing-extensions>=4.6.0", + "pymysql": "pymysql ; extra == 'pymysql'", + "one": "one<=0.4.17", + "two": "two^=0.4.17", + "three": "three~=0.4.17", + } + + for k, v in expected.items(): + assert k == distributions.core.parse_sub_package_name(v) + + +def test_distribution_sub_packages(): + dists = list(importlib.metadata.distributions()) + + res = distributions.distributions_sub_packages(dists) + + expected_dist = "typer" + expected_sub_package = "typing-extensions" + + assert res.get(expected_dist) is not None + assert expected_sub_package in res[expected_dist] From 7c69613ae90e2eb92afda882fce70b86e29b47d9 Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 16 Jan 2024 10:57:12 +0100 Subject: [PATCH 4/5] feat(cli): gather package sub-dependencies when running poly check --- components/polylith/commands/check.py | 39 ++++++++++--------- components/polylith/distributions/core.py | 4 +- .../polylith/commands/test_check.py | 12 ++++++ .../components/polylith/commands/test_info.py | 5 --- 4 files changed, 35 insertions(+), 25 deletions(-) create mode 100644 test/components/polylith/commands/test_check.py delete mode 100644 test/components/polylith/commands/test_info.py diff --git a/components/polylith/commands/check.py b/components/polylith/commands/check.py index 267b2aea..9dc6a6f3 100644 --- a/components/polylith/commands/check.py +++ b/components/polylith/commands/check.py @@ -1,39 +1,42 @@ import importlib.metadata from pathlib import Path - +from typing import Set from polylith import alias, check, distributions -def run(root: Path, ns: str, project_data: dict, options: dict) -> bool: - is_verbose = options["verbose"] - is_quiet = options["quiet"] - is_strict = options["strict"] - library_alias = options["alias"] - +def collect_known_aliases_and_sub_dependencies( + project_data: dict, options: dict +) -> Set[str]: third_party_libs = project_data["deps"] - name = project_data["name"] + library_alias = options["alias"] - collected_imports = check.report.collect_all_imports(root, ns, project_data) dists = list(importlib.metadata.distributions()) - known_aliases = distributions.distributions_packages(dists) - known_aliases.update(alias.parse(library_alias)) - - """ - WIP: dist_packages = distributions.distributions_packages(dists) sub_packages = distributions.distributions_sub_packages(dists) custom_aliases = alias.parse(library_alias) - """ - extra = alias.pick(known_aliases, third_party_libs) + a = alias.pick(dist_packages, third_party_libs) + b = alias.pick(sub_packages, third_party_libs) + c = alias.pick(custom_aliases, third_party_libs) - libs = third_party_libs.union(extra) + return third_party_libs.union(a, b, c) + + +def run(root: Path, ns: str, project_data: dict, options: dict) -> bool: + is_verbose = options["verbose"] + is_quiet = options["quiet"] + is_strict = options["strict"] + + name = project_data["name"] + + collected_imports = check.report.collect_all_imports(root, ns, project_data) + collected_libs = collect_known_aliases_and_sub_dependencies(project_data, options) details = check.report.create_report( project_data, collected_imports, - libs, + collected_libs, is_strict, ) diff --git a/components/polylith/distributions/core.py b/components/polylith/distributions/core.py index 124c5ff9..29e71e9b 100644 --- a/components/polylith/distributions/core.py +++ b/components/polylith/distributions/core.py @@ -17,9 +17,9 @@ def dist_subpackages(dist) -> dict: name = dist.metadata["name"] dependencies = importlib.metadata.requires(name) or [] - parsed_package_names = {parse_sub_package_name(d) for d in dependencies} + parsed_package_names = list({parse_sub_package_name(d) for d in dependencies}) - return {name: parsed_package_names} + return {name: parsed_package_names} if dependencies else {} def map_sub_packages(acc, dist) -> dict: diff --git a/test/components/polylith/commands/test_check.py b/test/components/polylith/commands/test_check.py new file mode 100644 index 00000000..ecae22f7 --- /dev/null +++ b/test/components/polylith/commands/test_check.py @@ -0,0 +1,12 @@ +from polylith.commands import check + + +def test_collect_known_aliases_and_sub_dependencies(): + fake_project_data = {"deps": {"typer", "hello-world-library"}} + fake_options = {"alias": ["hello-world-library=hello"]} + + res = check.collect_known_aliases_and_sub_dependencies(fake_project_data, fake_options) + + assert "typer" in res + assert "typing-extensions" in res + assert "hello" in res diff --git a/test/components/polylith/commands/test_info.py b/test/components/polylith/commands/test_info.py deleted file mode 100644 index b5663bc6..00000000 --- a/test/components/polylith/commands/test_info.py +++ /dev/null @@ -1,5 +0,0 @@ -from polylith.commands import info - - -def test_sample(): - assert info is not None From d32003d47b470f817db3c768af59b31ca4d1495e Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 16 Jan 2024 13:19:00 +0100 Subject: [PATCH 5/5] feat(cli): bump version to 0.2.0 --- projects/polylith-cli/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/polylith-cli/pyproject.toml b/projects/polylith-cli/pyproject.toml index a042f613..c352e782 100644 --- a/projects/polylith-cli/pyproject.toml +++ b/projects/polylith-cli/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polylith-cli" -version = "0.1.0" +version = "0.2.0" description = "Python tooling support for the Polylith Architecture" authors = ['David Vujic'] homepage = "https://davidvujic.github.io/python-polylith-docs/"