From 1f683a240304a2fd301eb984d68e44423a9556ee Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sat, 4 Mar 2023 13:19:41 +0100 Subject: [PATCH 1/8] wip: poly check v2 --- components/polylith/check/grouping.py | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 components/polylith/check/grouping.py diff --git a/components/polylith/check/grouping.py b/components/polylith/check/grouping.py new file mode 100644 index 00000000..402c18a9 --- /dev/null +++ b/components/polylith/check/grouping.py @@ -0,0 +1,56 @@ +from pathlib import Path +from typing import Set + +from polylith import workspace +from polylith.libs.imports import list_imports + + +def fetch_all_imports(paths: Set[Path]) -> dict: + rows = [{p.name: list_imports(p)} for p in paths] + + return {k: v for row in rows for k, v in row.items()} + + +def extract_top_ns_from_imports(imports: Set[str]) -> Set: + return {imp.split(".")[0] for imp in imports} + + +def extract_top_ns(import_data: dict) -> dict: + return {k: extract_top_ns_from_imports(v) for k, v in import_data.items()} + + +def only_brick_imports(imports: Set[str], top_ns: str) -> Set[str]: + return {i for i in imports if i.startswith(top_ns)} + + +def only_bricks(import_data: dict, top_ns: str) -> dict: + return {k: only_brick_imports(v, top_ns) for k, v in import_data.items()} + + +def brick_import_to_name(brick_import: str) -> str: + parts = brick_import.split(".") + + return f"{parts[0]}.{parts[1]}" if len(parts) > 1 else brick_import + + +def only_brick_name(brick_imports: Set[str]) -> Set[str]: + return {brick_import_to_name(i) for i in brick_imports} + + +def only_brick_names(import_data: dict) -> dict: + return {k: only_brick_name(v) for k, v in import_data.items() if v} + + +def exclude_empty(import_data: dict) -> dict: + return {k: v for k, v in import_data.items() if v} + + +def get_brick_imports(root: Path, paths: Set[Path]) -> dict: + top_ns = workspace.parser.get_namespace_from_config(root) + + all_imports = fetch_all_imports(paths) + + with_only_bricks = only_bricks(all_imports, top_ns) + with_only_brick_names = only_brick_names(with_only_bricks) + + return exclude_empty(with_only_brick_names) From 4455e57c5c48d72f8c6a82599e4672d8cd13185f Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sat, 4 Mar 2023 15:57:03 +0100 Subject: [PATCH 2/8] wip: poly check v2 --- components/polylith/check/core.py | 25 --------- components/polylith/check/grouping.py | 36 +++---------- components/polylith/check/report.py | 55 +++++++++++++++----- components/polylith/libs/__init__.py | 13 ++++- components/polylith/libs/grouping.py | 13 +++-- components/polylith/libs/report.py | 6 +-- components/polylith/poetry/commands/check.py | 54 +++++++++++++++++-- 7 files changed, 123 insertions(+), 79 deletions(-) delete mode 100644 components/polylith/check/core.py diff --git a/components/polylith/check/core.py b/components/polylith/check/core.py deleted file mode 100644 index 6dc832c3..00000000 --- a/components/polylith/check/core.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import subprocess -from pathlib import Path -from typing import List - - -def navigate_to(path: Path): - os.chdir(str(path)) - - -def run_command(project_path: Path) -> List[str]: - current_dir = Path.cwd() - - navigate_to(project_path) - - try: - res = subprocess.run( - ["poetry", "check-project"], capture_output=True, text=True - ) - finally: - navigate_to(current_dir) - - res.check_returncode() - - return res.stdout.splitlines() diff --git a/components/polylith/check/grouping.py b/components/polylith/check/grouping.py index 402c18a9..2e39a520 100644 --- a/components/polylith/check/grouping.py +++ b/components/polylith/check/grouping.py @@ -1,22 +1,4 @@ -from pathlib import Path -from typing import Set - -from polylith import workspace -from polylith.libs.imports import list_imports - - -def fetch_all_imports(paths: Set[Path]) -> dict: - rows = [{p.name: list_imports(p)} for p in paths] - - return {k: v for row in rows for k, v in row.items()} - - -def extract_top_ns_from_imports(imports: Set[str]) -> Set: - return {imp.split(".")[0] for imp in imports} - - -def extract_top_ns(import_data: dict) -> dict: - return {k: extract_top_ns_from_imports(v) for k, v in import_data.items()} +from typing import Set, Union def only_brick_imports(imports: Set[str], top_ns: str) -> Set[str]: @@ -27,14 +9,16 @@ def only_bricks(import_data: dict, top_ns: str) -> dict: return {k: only_brick_imports(v, top_ns) for k, v in import_data.items()} -def brick_import_to_name(brick_import: str) -> str: +def brick_import_to_name(brick_import: str) -> Union[str, None]: parts = brick_import.split(".") - return f"{parts[0]}.{parts[1]}" if len(parts) > 1 else brick_import + return parts[1] if len(parts) > 1 else None + +def only_brick_name(brick_imports: Set[str]) -> Set: + res = {brick_import_to_name(i) for i in brick_imports} -def only_brick_name(brick_imports: Set[str]) -> Set[str]: - return {brick_import_to_name(i) for i in brick_imports} + return {i for i in res if i} def only_brick_names(import_data: dict) -> dict: @@ -45,11 +29,7 @@ def exclude_empty(import_data: dict) -> dict: return {k: v for k, v in import_data.items() if v} -def get_brick_imports(root: Path, paths: Set[Path]) -> dict: - top_ns = workspace.parser.get_namespace_from_config(root) - - all_imports = fetch_all_imports(paths) - +def extract_brick_imports(all_imports: dict, top_ns) -> dict: with_only_bricks = only_bricks(all_imports, top_ns) with_only_brick_names = only_brick_names(with_only_bricks) diff --git a/components/polylith/check/report.py b/components/polylith/check/report.py index 11dd12b4..5d034d1e 100644 --- a/components/polylith/check/report.py +++ b/components/polylith/check/report.py @@ -1,4 +1,8 @@ -from polylith.check.core import run_command +from pathlib import Path +from typing import Set + +from polylith import libs, workspace +from polylith.check import grouping from rich.console import Console from rich.theme import Theme @@ -12,22 +16,47 @@ ) -def run(project_data: dict) -> bool: +def print_missing_deps(brick_imports: dict, deps: Set[str], project_name: str) -> bool: + diff = libs.report.calculate_diff(brick_imports, deps) + + if not diff: + return True + console = Console(theme=info_theme) - project_name = project_data["name"] - project_path = project_data["path"] + missing = ", ".join(sorted(diff)) + + console.print(f":thinking_face: Cannot locate {missing} in {project_name}") + return False + + +def print_report( + root: Path, ns: str, project_data: dict, third_party_libs: Set +) -> bool: + name = project_data["name"] - with console.status(f"checking [proj]{project_name}[/]", spinner="monkey"): - result = run_command(project_path) + bases = {b for b in project_data.get("bases", [])} + components = {c for c in project_data.get("components", [])} - message = ["[proj]", project_name, "[/]", " "] - extra = [":warning:"] if result else [":heavy_check_mark:"] + bases_paths = workspace.paths.collect_bases_paths(root, ns, bases) + components_paths = workspace.paths.collect_components_paths(root, ns, components) + + all_imports_in_bases = libs.fetch_all_imports(bases_paths) + all_imports_in_components = libs.fetch_all_imports(components_paths) + + brick_imports = { + "bases": grouping.extract_brick_imports(all_imports_in_bases, ns), + "components": grouping.extract_brick_imports(all_imports_in_components, ns), + } + + third_party_imports = { + "bases": libs.extract_third_party_imports(all_imports_in_bases, ns), + "components": libs.extract_third_party_imports(all_imports_in_components, ns), + } - output = "".join(message + extra) - console.print(output) + packages = set().union(bases, components) - for row in result: - console.print(f"[data]{row}[/]") + brick_result = print_missing_deps(brick_imports, packages, name) + libs_result = print_missing_deps(third_party_imports, third_party_libs, name) - return True if not result else False + return all([brick_result, libs_result]) diff --git a/components/polylith/libs/__init__.py b/components/polylith/libs/__init__.py index 1581ce3a..eec89fd0 100644 --- a/components/polylith/libs/__init__.py +++ b/components/polylith/libs/__init__.py @@ -1,4 +1,13 @@ -from polylith.libs.grouping import get_third_party_imports from polylith.libs import report +from polylith.libs.grouping import ( + extract_third_party_imports, + fetch_all_imports, + get_third_party_imports, +) -__all__ = ["get_third_party_imports", "report"] +__all__ = [ + "report", + "extract_third_party_imports", + "fetch_all_imports", + "get_third_party_imports", +] diff --git a/components/polylith/libs/grouping.py b/components/polylith/libs/grouping.py index 785ed349..0652d653 100644 --- a/components/polylith/libs/grouping.py +++ b/components/polylith/libs/grouping.py @@ -37,14 +37,19 @@ def exclude_empty(import_data: dict) -> dict: return {k: v for k, v in import_data.items() if v} -def get_third_party_imports(root: Path, paths: Set[Path]) -> dict: +def extract_third_party_imports(all_imports: dict, top_ns: str) -> dict: python_version = get_python_version() std_libs = get_standard_libs(python_version) - top_ns = workspace.parser.get_namespace_from_config(root) - all_imports = fetch_all_imports(paths) top_level_imports = extract_top_ns(all_imports) - with_third_party = exclude_libs(top_level_imports, std_libs.union({top_ns})) return exclude_empty(with_third_party) + + +def get_third_party_imports(root: Path, paths: Set[Path]) -> dict: + top_ns = workspace.parser.get_namespace_from_config(root) + + all_imports = fetch_all_imports(paths) + + return extract_third_party_imports(all_imports, top_ns) diff --git a/components/polylith/libs/report.py b/components/polylith/libs/report.py index 1b70e06f..e312a47d 100644 --- a/components/polylith/libs/report.py +++ b/components/polylith/libs/report.py @@ -36,13 +36,13 @@ def flatten_imports(brick_imports: dict, brick: str) -> Set[str]: return set().union(*brick_imports.get(brick, {}).values()) -def calculate_diff(brick_imports: dict, third_party_libs: Set[str]) -> Set[str]: +def calculate_diff(brick_imports: dict, deps: Set[str]) -> Set[str]: bases_imports = flatten_imports(brick_imports, "bases") components_imports = flatten_imports(brick_imports, "components") - normalized_libs = {t.replace("-", "_") for t in third_party_libs} + normalized_deps = {t.replace("-", "_") for t in deps} - return set().union(bases_imports, components_imports).difference(normalized_libs) + return set().union(bases_imports, components_imports).difference(normalized_deps) def print_libs_summary(brick_imports: dict, project_name: str) -> None: diff --git a/components/polylith/poetry/commands/check.py b/components/polylith/poetry/commands/check.py index ee5e018d..991d64bd 100644 --- a/components/polylith/poetry/commands/check.py +++ b/components/polylith/poetry/commands/check.py @@ -1,22 +1,68 @@ from pathlib import Path +from typing import List, Set, Union from poetry.console.commands.command import Command -from polylith import check, project, repo +from poetry.factory import Factory +from polylith import check, info, project, repo, workspace + + +def get_projects_data(root: Path, ns: str) -> List[dict]: + bases = info.get_bases(root, ns) + components = info.get_components(root, ns) + + return info.get_bricks_in_projects(root, components, bases, ns) class CheckCommand(Command): name = "poly check" description = "Validates the Polylith workspace." + def find_third_party_libs(self, path: Union[Path, None]) -> Set: + project_poetry = Factory().create_poetry(path) if path else self.poetry + + if not project_poetry.locker.is_locked(): + raise ValueError("poetry.lock not found. Run `poetry lock` to create it.") + + packages = project_poetry.locker.locked_repository().packages + + return {p.name for p in packages} + + def print_report(self, root: Path, ns: str, project_data: dict) -> bool: + path = project_data["path"] + name = project_data["name"] + + try: + third_party_libs = self.find_third_party_libs(path) + return check.report.print_report(root, ns, project_data, third_party_libs) + except ValueError as e: + self.line_error(f"{name}: {e}") + return False + def handle(self) -> int: root = repo.find_workspace_root(Path.cwd()) + if not root: raise ValueError( "Didn't find the workspace root. Expected to find a workspace.toml file." ) - projects = project.get_project_names_and_paths(root) + ns = workspace.parser.get_namespace_from_config(root) + + projects_data = get_projects_data(root, ns) + + if self.option("directory"): + project_name = project.get_project_name(self.poetry.pyproject.data) + + data = next((p for p in projects_data if p["name"] == project_name), None) + + if not data: + raise ValueError(f"Didn't find project in {self.option('directory')}") + + res = self.print_report(root, ns, data) + result_code = 0 if res else 1 + else: + results = {self.print_report(root, ns, data) for data in projects_data} - res = [check.report.run(proj) for proj in projects] + result_code = 0 if all(results) else 1 - return 0 if all(res) else 1 + return result_code From bc04ddbfe3644b94d4288ca487bcc9c249bd632e Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sat, 4 Mar 2023 15:57:32 +0100 Subject: [PATCH 3/8] fix: accidental debug print statement --- components/polylith/poetry/commands/libs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/polylith/poetry/commands/libs.py b/components/polylith/poetry/commands/libs.py index 73d39c62..6a676264 100644 --- a/components/polylith/poetry/commands/libs.py +++ b/components/polylith/poetry/commands/libs.py @@ -60,7 +60,6 @@ def handle(self) -> int: projects_data = get_projects_data(root, ns) if self.option("directory"): - self.line("in directory!") project_name = project.get_project_name(self.poetry.pyproject.data) data = next((p for p in projects_data if p["name"] == project_name), None) From c8f012e15336ecf9a5b221646d1306b483e433ef Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sat, 4 Mar 2023 16:37:39 +0100 Subject: [PATCH 4/8] fix(parse imports): extract node names when parsing 'from x import x, y z' --- components/polylith/libs/imports.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/components/polylith/libs/imports.py b/components/polylith/libs/imports.py index 6facb6ca..cff943b7 100644 --- a/components/polylith/libs/imports.py +++ b/components/polylith/libs/imports.py @@ -7,8 +7,16 @@ def parse_import(node: ast.Import) -> List[str]: return [name.name for name in node.names] +def extract_import_from(node: ast.ImportFrom) -> List: + return ( + [f"{node.module}.{name.name}" for name in node.names] + if node.names + else [node.module] + ) + + def parse_import_from(node: ast.ImportFrom) -> List[str]: - return [node.module] if node.module and node.level == 0 else [] + return extract_import_from(node) if node.module and node.level == 0 else [] def parse_imports(node: ast.AST) -> List[str]: From f089b53f2b993e5b8bbfd227462c6e8d95393182 Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sat, 4 Mar 2023 16:49:06 +0100 Subject: [PATCH 5/8] docs: add docs about the poetry poly check command --- projects/poetry_polylith_plugin/README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/projects/poetry_polylith_plugin/README.md b/projects/poetry_polylith_plugin/README.md index d41651ea..daf6c492 100644 --- a/projects/poetry_polylith_plugin/README.md +++ b/projects/poetry_polylith_plugin/README.md @@ -98,6 +98,7 @@ Useful for CI: poetry poly diff --short ``` + #### Libs Show info about the third-party libraries used in the workspace: @@ -117,16 +118,23 @@ The very nice dependency lookup features of `Poetry` is used behind the scenes b Show info about libraries used in a specific project. - #### Check -Validates the Polylith workspace: +Validates the Polylith workspace, checking for any missing dependencies (bricks and third-party libraries): ``` shell poetry poly check ``` -**NOTE**: this feature is built on top of the `poetry check-project` command from the [Multiproject](https://github.com/DavidVujic/poetry-multiproject-plugin) plugin. -Make sure that you have the latest version of poetry-multiproject-plugin installed to be able to use the `poly check` command. +**NOTE**: this feature is built on top of the `poetry poly libs` command, +and (just like the `poetry poly libs` command) it expects a `poetry.lock` of a project to be present. +If missing, there is a Poetry command available: `poetry lock --directory path/to-project`. + + +##### Options +`--directory` or `-C` + +Show info about libraries used in a specific project. + #### Testing The `create` commands will also create corresponding unit tests. It is possible to disable thi behaviour From d37a4992ef74630dd92025556a5bbdacc9893da9 Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sat, 4 Mar 2023 16:49:36 +0100 Subject: [PATCH 6/8] bump version to 1.4.0 --- projects/poetry_polylith_plugin/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/poetry_polylith_plugin/pyproject.toml b/projects/poetry_polylith_plugin/pyproject.toml index ac064e70..0ffcc860 100644 --- a/projects/poetry_polylith_plugin/pyproject.toml +++ b/projects/poetry_polylith_plugin/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "poetry-polylith-plugin" -version = "1.3.1" +version = "1.4.0" description = "A Poetry plugin that adds tooling support for the Polylith Architecture" authors = ["David Vujic"] homepage = "https://github.com/davidvujic/python-polylith" From d7d1a1f9b4f897f171df3f0f06130e2bcb3536b7 Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sat, 4 Mar 2023 16:54:56 +0100 Subject: [PATCH 7/8] docs: fix typo --- projects/poetry_polylith_plugin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/poetry_polylith_plugin/README.md b/projects/poetry_polylith_plugin/README.md index daf6c492..93c433d0 100644 --- a/projects/poetry_polylith_plugin/README.md +++ b/projects/poetry_polylith_plugin/README.md @@ -137,7 +137,7 @@ Show info about libraries used in a specific project. #### Testing -The `create` commands will also create corresponding unit tests. It is possible to disable thi behaviour +The `create` commands will also create corresponding unit tests. It is possible to disable this behaviour by setting `enabled = false` in the `workspace.toml` file. From 9a9f198500171db6c8a59d8945636ecc05d4a780 Mon Sep 17 00:00:00 2001 From: davidvujic Date: Sun, 5 Mar 2023 10:04:54 +0100 Subject: [PATCH 8/8] fix(parse import): iterate ast node names that are of type alias --- components/polylith/libs/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/polylith/libs/imports.py b/components/polylith/libs/imports.py index cff943b7..8eb46fa2 100644 --- a/components/polylith/libs/imports.py +++ b/components/polylith/libs/imports.py @@ -9,7 +9,7 @@ def parse_import(node: ast.Import) -> List[str]: def extract_import_from(node: ast.ImportFrom) -> List: return ( - [f"{node.module}.{name.name}" for name in node.names] + [f"{node.module}.{alias.name}" for alias in node.names] if node.names else [node.module] )