From 285ba12c98b8dacb5354dbc35614c19b57227a9e Mon Sep 17 00:00:00 2001 From: Nat Noordanus Date: Sat, 3 Dec 2022 20:50:32 +0100 Subject: [PATCH] Support providing a cwd for included tasks #110 --- README.rst | 30 ++++++-- poethepoet/config.py | 71 ++++++++++++++----- .../fixtures/monorepo_project/pyproject.toml | 11 +++ .../subproject_1/pyproject.toml | 6 ++ .../subproject_2/extra_tasks.toml | 5 ++ .../subproject_2/pyproject.toml | 7 ++ tests/test_includes.py | 52 +++++++++++++- 7 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 tests/fixtures/monorepo_project/pyproject.toml create mode 100644 tests/fixtures/monorepo_project/subproject_1/pyproject.toml create mode 100644 tests/fixtures/monorepo_project/subproject_2/extra_tasks.toml create mode 100644 tests/fixtures/monorepo_project/subproject_2/pyproject.toml diff --git a/README.rst b/README.rst index e5d18976..3e50e8c2 100644 --- a/README.rst +++ b/README.rst @@ -936,12 +936,12 @@ to use *fish* by default. Load tasks from another file ============================ -There are some scenarios where one might wish to define tasks outside of pyproject.toml. -For example, if you want to share tasks between projects via git modules, generate tasks -definitions dynamically, or simply have a lot of tasks and don't want the pyproject.toml -to get too large. This can be achieved by creating a toml or json file within your -project directory structure including the same structure for tasks as used in -pyproject.toml +There are some scenarios where one might wish to define tasks outside of pyproject.toml, +or to collect tasks from multiple projects into one. For example, if you want to share +tasks between projects via git modules, generate tasks definitions dynamically, organise +your code in a monorepo, or simply have a lot of tasks and don't want the pyproject.toml +to get too large. This can be achieved by creating a toml or json including the same +structure for tasks as used in pyproject.toml For example: @@ -972,6 +972,24 @@ so: Files are loaded in the order specified. If an item already exists then the included value it ignored. +If an included task file itself includes other files, these second order includes are +not inherited, so circular includes don't cause any problems. + +When including files from another location, you can also specify that tasks from that +other file should be run from within a specific directory. For example with the +following configuration, when tasks imported from my_subproject are run +from the root, the task will actually execute as if it had been run from the +my_subproject subdirectory. + +.. code-block:: toml + + [[tool.poe.include]] + path = "my_subproject/pyproject.toml" + cwd = "my_subproject" + +The cwd option still has the limitation that it cannot be used to specify a directory +outside of parent directory of the pyproject.toml file that poe is running with. + If a referenced file is missing then poe ignores it without error, though failure to read the contents will result in failure. diff --git a/poethepoet/config.py b/poethepoet/config.py index 6f7abce1..f10d771b 100644 --- a/poethepoet/config.py +++ b/poethepoet/config.py @@ -6,7 +6,7 @@ except ImportError: import tomli # type: ignore -from typing import Any, Dict, Mapping, Optional, Sequence, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union from .exceptions import PoeException @@ -36,7 +36,7 @@ class PoeConfig: "env": dict, "envfile": (str, list), "executor": dict, - "include": (str, list), + "include": (str, list, dict), "poetry_command": str, "poetry_hooks": dict, "shell_interpreter": (str, list), @@ -230,15 +230,31 @@ def find_pyproject_toml(self, target_dir: Optional[str] = None) -> Path: return maybe_result def _load_includes(self, project_dir: Path): - includes: Union[str, Sequence[str]] = self._poe.get("include", tuple()) - if isinstance(includes, str): - includes = (includes,) - - for path_str in includes: - if not isinstance(path_str, str): - raise PoeException(f"Invalid include value {path_str!r}") + include_option: Union[str, Sequence[str]] = self._poe.get("include", tuple()) + includes: List[Dict[str, str]] = [] + + if isinstance(include_option, str): + includes.append({"path": include_option}) + elif isinstance(include_option, dict): + includes.append(include_option) + elif isinstance(include_option, list): + valid_keys = {"path", "cwd"} + for include in include_option: + if isinstance(include, str): + includes.append({"path": include}) + elif ( + isinstance(include, dict) + and include.get("path") + and set(include.keys()) <= valid_keys + ): + includes.append(include) + else: + raise PoeException( + f"Invalid item for the include option {include!r}" + ) - include_path = project_dir.joinpath(path_str).resolve() + for include in includes: + include_path = project_dir.joinpath(include["path"]).resolve() if not include_path.exists(): # TODO: print warning in verbose mode, requires access to ui somehow @@ -247,7 +263,9 @@ def _load_includes(self, project_dir: Path): if include_path.name.endswith(".toml"): try: with include_path.open("rb") as file: - self._merge_config(tomli.load(file), include_path) + self._merge_config( + tomli.load(file), include_path, include.get("cwd") + ) except tomli.TOMLDecodeError as error: raise PoeException( f"Couldn't parse included toml file from {include_path}", error @@ -256,21 +274,26 @@ def _load_includes(self, project_dir: Path): elif include_path.name.endswith(".json"): try: with include_path.open("rb") as file: - self._merge_config(json.load(file), include_path) + self._merge_config( + json.load(file), include_path, include.get("cwd") + ) except json.decoder.JSONDecodeError as error: raise PoeException( f"Couldn't parse included json file from {include_path}", error ) from error - def _merge_config(self, extra_config: Mapping[str, Any], path: Path): - from .task import PoeTask - + def _merge_config( + self, + extra_config: Mapping[str, Any], + include_path: Path, + task_cwd: Optional[str], + ): try: poe_config = extra_config.get("tool", {}).get("poe", {}) tasks = poe_config.get("tasks", {}) except AttributeError as error: raise PoeException( - f"Invalid content in included file from {path}", error + f"Invalid content in included file from {include_path}", error ) from error # Env is special because it can be extended rather than just overwritten @@ -283,6 +306,22 @@ def _merge_config(self, extra_config: Mapping[str, Any], path: Path): # Includes additional tasks with preserved ordering self._poe["tasks"] = own_tasks = self._poe.get("tasks", {}) for task_name, task_def in tasks.items(): + + if task_cwd: + # Override the config of each task to use the include level cwd as a + # base for the task level cwd + if "cwd" in task_def: + # rebase the configured cwd onto the include level cwd + task_def["cwd"] = ( + Path(self.project_dir) + .joinpath(task_cwd) + .resolve() + .joinpath(task_def["cwd"]) + .relative_to(self.project_dir) + ) + else: + task_def["cwd"] = task_cwd + if task_name not in own_tasks: own_tasks[task_name] = task_def diff --git a/tests/fixtures/monorepo_project/pyproject.toml b/tests/fixtures/monorepo_project/pyproject.toml new file mode 100644 index 00000000..70cbd587 --- /dev/null +++ b/tests/fixtures/monorepo_project/pyproject.toml @@ -0,0 +1,11 @@ + +[[tool.poe.include]] +path = "subproject_1/pyproject.toml" +[[tool.poe.include]] +path = "subproject_2/pyproject.toml" +cwd = "subproject_2" + + +[tool.poe.tasks.get_cwd_0] +interpreter = "python" +shell = "import os; print(os.getcwd())" diff --git a/tests/fixtures/monorepo_project/subproject_1/pyproject.toml b/tests/fixtures/monorepo_project/subproject_1/pyproject.toml new file mode 100644 index 00000000..c02d844f --- /dev/null +++ b/tests/fixtures/monorepo_project/subproject_1/pyproject.toml @@ -0,0 +1,6 @@ + + + +[tool.poe.tasks.get_cwd_1] +interpreter = "python" +shell = "import os; print(os.getcwd())" diff --git a/tests/fixtures/monorepo_project/subproject_2/extra_tasks.toml b/tests/fixtures/monorepo_project/subproject_2/extra_tasks.toml new file mode 100644 index 00000000..a07e2c37 --- /dev/null +++ b/tests/fixtures/monorepo_project/subproject_2/extra_tasks.toml @@ -0,0 +1,5 @@ + + +[tool.poe.tasks.extra_task] +interpreter = "python" +shell = "print('nope')" diff --git a/tests/fixtures/monorepo_project/subproject_2/pyproject.toml b/tests/fixtures/monorepo_project/subproject_2/pyproject.toml new file mode 100644 index 00000000..928194d9 --- /dev/null +++ b/tests/fixtures/monorepo_project/subproject_2/pyproject.toml @@ -0,0 +1,7 @@ + +tool.poe.include = ["extra_tasks.toml", { path = "../pyproject.toml" }] + +[tool.poe.tasks.get_cwd_2] +interpreter = "python" +shell = "import os; print(os.getcwd())" + diff --git a/tests/test_includes.py b/tests/test_includes.py index fc85fd7b..2e728f66 100644 --- a/tests/test_includes.py +++ b/tests/test_includes.py @@ -67,7 +67,7 @@ def test_running_from_multiple_includes(run_poe_subproc, projects): assert result.stderr == "" -def test_docs_for_onlyincludes(run_poe_subproc, projects): +def test_docs_for_only_includes(run_poe_subproc, projects): result = run_poe_subproc( f'--root={projects["includes/only_includes"]}', ) @@ -80,3 +80,53 @@ def test_docs_for_onlyincludes(run_poe_subproc, projects): ) in result.capture assert result.stdout == "" assert result.stderr == "" + + +def test_monorepo_contains_only_expected_tasks(run_poe_subproc, projects): + result = run_poe_subproc(project="monorepo") + assert result.capture.endswith( + "CONFIGURED TASKS\n" + " get_cwd_0 \n" + " get_cwd_1 \n" + " get_cwd_2 \n\n\n" + ) + assert result.stdout == "" + assert result.stderr == "" + + +def test_monorepo_can_also_include_parent(run_poe_subproc, projects): + result = run_poe_subproc(cwd=projects["monorepo/subproject_2"]) + assert result.capture.endswith( + "CONFIGURED TASKS\n" + " get_cwd_2 \n" + " extra_task \n" + " get_cwd_0 \n\n\n" + ) + assert result.stdout == "" + assert result.stderr == "" + + result = run_poe_subproc("get_cwd_0", cwd=projects["monorepo/subproject_2"]) + assert result.capture == "Poe => import os; print(os.getcwd())\n" + assert result.stdout.endswith( + "poethepoet/tests/fixtures/monorepo_project/subproject_2\n" + ) + assert result.stderr == "" + + +def test_monorepo_runs_each_task_with_expected_cwd(run_poe_subproc, projects): + result = run_poe_subproc("get_cwd_0", project="monorepo") + assert result.capture == "Poe => import os; print(os.getcwd())\n" + assert result.stdout.endswith("poethepoet/tests/fixtures/monorepo_project\n") + assert result.stderr == "" + + result = run_poe_subproc("get_cwd_1", project="monorepo") + assert result.capture == "Poe => import os; print(os.getcwd())\n" + assert result.stdout.endswith("poethepoet/tests/fixtures/monorepo_project\n") + assert result.stderr == "" + + result = run_poe_subproc("get_cwd_2", project="monorepo") + assert result.capture == "Poe => import os; print(os.getcwd())\n" + assert result.stdout.endswith( + "poethepoet/tests/fixtures/monorepo_project/subproject_2\n" + ) + assert result.stderr == ""