From 04b1b0aff7617febfcfece78e013289777c7fb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 8 May 2023 16:22:29 +0200 Subject: [PATCH] Migrate default project template to static project metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uses `requirements.txt` for dev requirements in project template. Fix gh-2280. Fix gh-2519. Signed-off-by: Juan Luis Cano Rodríguez --- features/environment.py | 13 +++++-- features/steps/cli_steps.py | 10 ++--- kedro/framework/cli/micropkg.py | 4 +- kedro/framework/cli/project.py | 23 ++++++----- .../{{ cookiecutter.repo_name }}/README.md | 12 ------ .../pyproject.toml | 35 +++++++++++++++++ .../requirements.txt | 6 ++- .../{{ cookiecutter.repo_name }}/setup.py | 39 ------------------- tests/framework/cli/micropkg/conftest.py | 6 +-- tests/framework/cli/test_project.py | 26 ++++++------- tests/framework/cli/test_starters.py | 6 +-- 11 files changed, 86 insertions(+), 94 deletions(-) delete mode 100644 kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py diff --git a/features/environment.py b/features/environment.py index c98246dc85..6c8952e570 100644 --- a/features/environment.py +++ b/features/environment.py @@ -116,13 +116,18 @@ def _setup_minimal_env(context): def _install_project_requirements(context): install_reqs = ( - Path( - "kedro/templates/project/{{ cookiecutter.repo_name }}/src/requirements.txt" - ) + Path("kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt") .read_text(encoding="utf-8") .splitlines() ) - install_reqs = [req for req in install_reqs if "{" not in req] + install_reqs = [ + req + for req in install_reqs + if (req.strip()) + and ("{" not in req) + and (not req.startswith("-e")) + and (not req.startswith("#")) + ] install_reqs.append(".[pandas.CSVDataSet]") call([context.pip, "install", *install_reqs], env=context.env) return context diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 76bb0d2722..58e5c213af 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -162,7 +162,7 @@ def create_config_file(context): @given("I have installed the project dependencies") def pip_install_dependencies(context): """Install project dependencies using pip.""" - reqs_path = "src/requirements.txt" + reqs_path = "requirements.txt" res = run( [context.pip, "install", "-r", reqs_path], env=context.env, @@ -410,7 +410,7 @@ def update_kedro_req(context: behave.runner.Context): """Replace kedro as a standalone requirement with a line that includes all of kedro's dependencies (-r kedro/requirements.txt) """ - reqs_path = context.root_project_dir / "src" / "requirements.txt" + reqs_path = context.root_project_dir / "requirements.txt" kedro_reqs = f"-r {context.requirements_path.as_posix()}" if reqs_path.is_file(): @@ -428,7 +428,7 @@ def update_kedro_req(context: behave.runner.Context): @when("I add {dependency} to the requirements") def add_req(context: behave.runner.Context, dependency: str): - reqs_path = context.root_project_dir / "src" / "requirements.txt" + reqs_path = context.root_project_dir / "requirements.txt" if reqs_path.is_file(): reqs_path.write_text(reqs_path.read_text() + "\n" + str(dependency) + "\n") @@ -610,14 +610,14 @@ def check_docs_generated(context: behave.runner.Context): @then("requirements should be generated") def check_reqs_generated(context: behave.runner.Context): """Check that new project requirements are generated.""" - reqs_path = context.root_project_dir / "src" / "requirements.lock" + reqs_path = context.root_project_dir / "requirements.lock" assert reqs_path.is_file() assert "This file is autogenerated by pip-compile" in reqs_path.read_text() @then("{dependency} should be in the requirements") def check_dependency_in_reqs(context: behave.runner.Context, dependency: str): - reqs_path = context.root_project_dir / "src" / "requirements.txt" + reqs_path = context.root_project_dir / "requirements.txt" assert dependency in reqs_path.read_text() diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py index ce7d28fb8b..bd88f46d43 100644 --- a/kedro/framework/cli/micropkg.py +++ b/kedro/framework/cli/micropkg.py @@ -165,7 +165,7 @@ def _pull_package( package_reqs = re.findall(reqs_element_pattern, list_reqs[0]) if package_reqs: - requirements_txt = metadata.source_dir / "requirements.txt" + requirements_txt = metadata.project_path / "requirements.txt" _append_package_reqs(requirements_txt, package_reqs, package_name) _clean_pycache(temp_dir_path) @@ -741,7 +741,7 @@ def _generate_sdist_file( # Build a setup.py on the fly try: install_requires = _make_install_requires( - package_source / "requirements.txt" # type: ignore + metadata.project_path / "requirements.txt" # type: ignore ) except Exception as exc: click.secho("FAILED", fg="red") diff --git a/kedro/framework/cli/project.py b/kedro/framework/cli/project.py index 73f4bfc912..47270db122 100644 --- a/kedro/framework/cli/project.py +++ b/kedro/framework/cli/project.py @@ -88,9 +88,9 @@ def test(metadata: ProjectMetadata, args, **kwargs): # pylint: disable=unused-a try: _check_module_importable("pytest") except KedroCliError as exc: - source_path = metadata.source_dir + project_path = metadata.project_path raise KedroCliError( - NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path)) + NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(project_path)) ) from exc python_call("pytest", args) @@ -110,12 +110,14 @@ def lint( click.secho(deprecation_message, fg="red") source_path = metadata.source_dir + project_path = metadata.project_path package_name = metadata.package_name - files = files or (str(source_path / "tests"), str(source_path / package_name)) + files = files or (str(project_path / "tests"), str(source_path / package_name)) if "PYTHONPATH" not in os.environ: # isort needs the source path to be in the 'PYTHONPATH' environment # variable to treat it as a first-party import location + # NOTE: Actually, `pip install [-e] .` achieves the same os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover for module_name in ("flake8", "isort", "black"): @@ -123,7 +125,7 @@ def lint( _check_module_importable(module_name) except KedroCliError as exc: raise KedroCliError( - NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path)) + NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(project_path)) ) from exc python_call("black", ("--check",) + files if check_only else files) @@ -149,7 +151,7 @@ def ipython( @click.pass_obj # this will pass the metadata as first argument def package(metadata: ProjectMetadata): """Package the project as a Python wheel.""" - source_path = metadata.source_dir + project_path = metadata.project_path call( [ sys.executable, @@ -159,7 +161,7 @@ def package(metadata: ProjectMetadata): "--outdir", "../dist", ], - cwd=str(source_path), + cwd=str(project_path), ) directory = ( @@ -199,10 +201,11 @@ def build_docs(metadata: ProjectMetadata, open_docs): click.secho(deprecation_message, fg="red") source_path = metadata.source_dir + project_path = metadata.project_path package_name = metadata.package_name - python_call("pip", ["install", str(source_path / "[docs]")]) - python_call("pip", ["install", "-r", str(source_path / "requirements.txt")]) + python_call("pip", ["install", str(project_path / "[docs]")]) + python_call("pip", ["install", "-r", str(project_path / "requirements.txt")]) python_call("ipykernel", ["install", "--user", f"--name={package_name}"]) shutil.rmtree("docs/build", ignore_errors=True) call( @@ -291,7 +294,7 @@ def activate_nbstripout( ) click.secho(deprecation_message, fg="red") - source_path = metadata.source_dir + project_path = metadata.source_dir click.secho( ( "Notebook output cells will be automatically cleared before committing" @@ -304,7 +307,7 @@ def activate_nbstripout( _check_module_importable("nbstripout") except KedroCliError as exc: raise KedroCliError( - NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(source_path)) + NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(project_path)) ) from exc try: diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md b/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md index 9ece71ad8e..98f5ba29b1 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md @@ -64,12 +64,6 @@ After this, if you'd like to update your project requirements, please update `re > Jupyter, JupyterLab, and IPython are already included in the project requirements by default, so once you have run `pip install -r requirements.txt` you will not need to take any extra steps before you use them. ### Jupyter -To use Jupyter notebooks in your Kedro project, you need to install Jupyter: - -``` -pip install jupyter -``` - After installing Jupyter, you can start a local notebook server: ``` @@ -77,12 +71,6 @@ kedro jupyter notebook ``` ### JupyterLab -To use JupyterLab, you need to install it: - -``` -pip install jupyterlab -``` - You can also start JupyterLab: ``` diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml index 7ae06368bd..f39867c572 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml @@ -1,3 +1,38 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "{{ cookiecutter.python_package }}" +dependencies = [ + "kedro~={{ cookiecutter.kedro_version }}", +] +dynamic = ["version"] + +[project.scripts] +{{ cookiecutter.repo_name }} = "{{ cookiecutter.python_package }}.__main__:main" + +[project.optional-dependencies] +docs = [ + "docutils<0.18.0", + "sphinx~=3.4.3", + "sphinx_rtd_theme==0.5.1", + "nbsphinx==0.8.1", + "nbstripout~=0.4", + "sphinx-autodoc-typehints==1.11.1", + "sphinx_copybutton==0.3.1", + "ipykernel>=5.3, <7.0", + "Jinja2<3.1.0", + "myst-parser~=0.17.2", +] + +[tool.setuptools.dynamic] +version = {attr = "{{ cookiecutter.python_package }}.__version__"} + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false + [tool.kedro] package_name = "{{ cookiecutter.python_package }}" project_name = "{{ cookiecutter.project_name }}" diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt b/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt index aa7ee32014..50cf28b712 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt @@ -1,3 +1,7 @@ +# Install library code +-e file:. + +# Development dependencies black~=22.0 flake8>=3.7.9, <5.0 ipython>=7.31.1, <8.0; python_version < '3.8' @@ -6,8 +10,6 @@ isort~=5.0 jupyter~=1.0 jupyterlab_server>=2.11.1, <2.16.0 jupyterlab~=3.0, <3.6.0 -kedro~={{ cookiecutter.kedro_version }} -kedro-telemetry~=0.2.0 nbstripout~=0.4 pytest-cov~=3.0 pytest-mock>=1.7.1, <2.0 diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py deleted file mode 100644 index 8e62d661f8..0000000000 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -from setuptools import find_packages, setup - -entry_point = ( - "{{ cookiecutter.repo_name }} = {{ cookiecutter.python_package }}.__main__:main" -) - - -# get the dependencies and installs -with open("requirements.txt", encoding="utf-8") as f: - # Make sure we strip all comments and options (e.g "--extra-index-url") - # that arise from a modified pip.conf file that configure global options - # when running kedro build-reqs - requires = [] - for line in f: - req = line.split("#", 1)[0].strip() - if req and not req.startswith("--"): - requires.append(req) - -setup( - name="{{ cookiecutter.python_package }}", - version="0.1", - packages=find_packages(exclude=["tests"]), - entry_points={"console_scripts": [entry_point]}, - install_requires=requires, - extras_require={ - "docs": [ - "docutils<0.18.0", - "sphinx~=3.4.3", - "sphinx_rtd_theme==0.5.1", - "nbsphinx==0.8.1", - "nbstripout~=0.4", - "sphinx-autodoc-typehints==1.11.1", - "sphinx_copybutton==0.3.1", - "ipykernel>=5.3, <7.0", - "Jinja2<3.1.0", - "myst-parser~=0.17.2", - ] - }, -) diff --git a/tests/framework/cli/micropkg/conftest.py b/tests/framework/cli/micropkg/conftest.py index ff8348b755..faf7b13e91 100644 --- a/tests/framework/cli/micropkg/conftest.py +++ b/tests/framework/cli/micropkg/conftest.py @@ -26,7 +26,7 @@ def cleanup_micropackages(fake_repo_path, fake_package_path): if each.is_file(): each.unlink() - tests = fake_repo_path / "src" / "tests" / micropackage + tests = fake_repo_path / "tests" / micropackage if tests.is_dir(): shutil.rmtree(str(tests)) @@ -35,7 +35,7 @@ def cleanup_micropackages(fake_repo_path, fake_package_path): def cleanup_pipelines(fake_repo_path, fake_package_path): pipes_path = fake_package_path / "pipelines" old_pipelines = {p.name for p in pipes_path.iterdir() if p.is_dir()} - requirements_txt = fake_repo_path / "src" / "requirements.txt" + requirements_txt = fake_repo_path / "requirements.txt" requirements = requirements_txt.read_text() yield @@ -53,7 +53,7 @@ def cleanup_pipelines(fake_repo_path, fake_package_path): if each.is_file(): each.unlink() - tests = fake_repo_path / "src" / "tests" / "pipelines" / pipeline + tests = fake_repo_path / "tests" / "pipelines" / pipeline if tests.is_dir(): shutil.rmtree(str(tests)) diff --git a/tests/framework/cli/test_project.py b/tests/framework/cli/test_project.py index 92e0d024cd..dda8101fd3 100644 --- a/tests/framework/cli/test_project.py +++ b/tests/framework/cli/test_project.py @@ -121,7 +121,7 @@ def test_pytest_not_installed( fake_project_cli, ["test", "--random-arg", "value"], obj=fake_metadata ) expected_message = NO_DEPENDENCY_MESSAGE.format( - module="pytest", src=str(fake_repo_path / "src") + module="pytest", src=str(fake_repo_path) ) assert result.exit_code @@ -148,7 +148,7 @@ def test_lint( assert not result.exit_code, result.stdout expected_files = files or ( - str(fake_repo_path / "src/tests"), + str(fake_repo_path / "tests"), str(fake_repo_path / "src/dummy_package"), ) expected_calls = [ @@ -185,7 +185,7 @@ def test_lint_check_only( assert not result.exit_code, result.stdout expected_files = files or ( - str(fake_repo_path / "src/tests"), + str(fake_repo_path / "tests"), str(fake_repo_path / "src/dummy_package"), ) expected_calls = [ @@ -217,7 +217,7 @@ def test_import_not_installed( result = CliRunner().invoke(fake_project_cli, ["lint"], obj=fake_metadata) expected_message = NO_DEPENDENCY_MESSAGE.format( - module=module_name, src=str(fake_repo_path / "src") + module=module_name, src=str(fake_repo_path) ) assert result.exit_code, result.stdout @@ -351,10 +351,10 @@ def test_happy_path( ) python_call_mock.assert_has_calls( [ - mocker.call("pip", ["install", str(fake_repo_path / "src/[docs]")]), + mocker.call("pip", ["install", str(fake_repo_path / "[docs]")]), mocker.call( "pip", - ["install", "-r", str(fake_repo_path / "src/requirements.txt")], + ["install", "-r", str(fake_repo_path / "requirements.txt")], ), mocker.call("ipykernel", ["install", "--user", "--name=dummy_package"]), ] @@ -395,9 +395,9 @@ def test_compile_from_requirements_file( "piptools", [ "compile", - str(fake_repo_path / "src" / "requirements.txt"), + str(fake_repo_path / "requirements.txt"), "--output-file", - str(fake_repo_path / "src" / "requirements.lock"), + str(fake_repo_path / "requirements.lock"), ], ) @@ -410,10 +410,10 @@ def test_compile_from_input_and_to_output_file( fake_metadata, ): # File exists: - input_file = fake_repo_path / "src" / "dev-requirements.txt" + input_file = fake_repo_path / "dev-requirements.txt" with open(input_file, "a", encoding="utf-8") as file: file.write("") - output_file = fake_repo_path / "src" / "dev-requirements.lock" + output_file = fake_repo_path / "dev-requirements.lock" result = CliRunner().invoke( fake_project_cli, @@ -444,7 +444,7 @@ def test_extra_args( extra_args, fake_metadata, ): - requirements_txt = fake_repo_path / "src" / "requirements.txt" + requirements_txt = fake_repo_path / "requirements.txt" result = CliRunner().invoke( fake_project_cli, ["build-reqs"] + extra_args, obj=fake_metadata @@ -457,7 +457,7 @@ def test_extra_args( ["compile"] + extra_args + [str(requirements_txt)] - + ["--output-file", str(fake_repo_path / "src" / "requirements.lock")] + + ["--output-file", str(fake_repo_path / "requirements.lock")] ) python_call_mock.assert_called_once_with("piptools", call_args) @@ -466,7 +466,7 @@ def test_missing_requirements_txt( self, fake_project_cli, mocker, fake_metadata, os_name, fake_repo_path ): """Test error when input file requirements.txt doesn't exists.""" - requirements_txt = fake_repo_path / "src" / "requirements.txt" + requirements_txt = fake_repo_path / "requirements.txt" mocker.patch("kedro.framework.cli.project.os").name = os_name mocker.patch.object(Path, "is_file", return_value=False) diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index 26fc6ac3e5..37c0512110 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -17,7 +17,7 @@ KedroStarterSpec, ) -FILES_IN_TEMPLATE = 31 +FILES_IN_TEMPLATE = 30 @pytest.fixture @@ -70,9 +70,7 @@ def _assert_template_ok( assert (full_path / ".gitignore").is_file() assert project_name in (full_path / "README.md").read_text(encoding="utf-8") assert "KEDRO" in (full_path / ".gitignore").read_text(encoding="utf-8") - assert kedro_version in (full_path / "src" / "requirements.txt").read_text( - encoding="utf-8" - ) + assert kedro_version in (full_path / "pyproject.toml").read_text(encoding="utf-8") assert (full_path / "src" / python_package / "__init__.py").is_file()