[KED-2188] Drop support for .kedro.yml file in favour of pyproject.to…

…ml (#839)

RELEASE.md

@@ -3,6 +3,7 @@
 ## Major features and improvements
 * Introduced `KedroSession` which is responsible for managing the lifecycle of a Kedro run.
 * Added `kedro catalog create` command. It creates `<conf_root>/<env>/catalog/<pipeline_name>.yml` configuration file with `MemoryDataSet` datasets for each dataset in a registered pipeline if it is missing from Data Catalog.
+* Added `settings.py` and `pyproject.toml` (to replace `.kedro.yml`) for project configuration, in line with Python best practice.
 
 ## Bug fixes and other changes
 *  Bumped maximum required `fsspec` version to 0.8.
@@ -24,6 +25,8 @@
 * Added `DatasetSpecs` with hooks to run before and after loading and saving datasets from/to the catalog.
 * `kedro new --starter` now defaults to fetching the starter template matching the installed Kedro version.
 * Renamed `kedro_cli.py` to `cli.py` and moved it inside the Python packge (`src/<package_name>/`).
+* Removed `.kedro.yml` from the project template and replaced it with `pyproject.toml`.
+* `KEDRO_CONFIGS` constant (residing in `kedro.framework.context.context`) has been removed.
 
 ## Thanks for supporting contributions
 [Deepyaman Datta](https://github.com/deepyaman)

docs/conf.py

@@ -102,6 +102,7 @@
         "int",
         "float",
         "str",
+        "tuple",
         "Any",
         "Dict",
         "typing.Dict",

docs/source/14_api_docs/kedro.framework.context.rst

@@ -12,6 +12,7 @@ Base Classes
     :toctree:
     :template: autosummary/class.rst
 
+    kedro.framework.context.context.ProjectSettings
     kedro.framework.context.KedroContext
 
 Functions

features/load_context.feature

@@ -36,13 +36,13 @@ Feature: Custom Kedro project
 
     Scenario: Update the source directory to be nested
         When I move the package to "src/nested"
-        And Source directory is updated to "src/nested" in kedro.yml
+        And Source directory is updated to "src/nested" in pyproject.toml
         And I execute the kedro command "run"
         Then I should get a successful exit code
 
     Scenario: Update the source directory to be outside of src
         When I move the package to "."
-        And Source directory is updated to "." in kedro.yml
+        And Source directory is updated to "." in pyproject.toml
         And I execute the kedro command "run"
         Then I should get a successful exit code
 

features/steps/cli_steps.py

@@ -37,6 +37,7 @@
 
 import behave
 import requests
+import toml
 import yaml
 from behave import given, then, when
 
@@ -262,13 +263,13 @@ def install_test_plugin(context):
 
 @given('I have disabled hooks for "{plugin}" plugin via config')
 def disable_plugin_hooks(context, plugin):
-    """Set `disable_hooks_for_plugins` in `.kedro.yml`."""
-    kedro_yml_path = context.root_project_dir / ".kedro.yml"
-
-    with kedro_yml_path.open("r+") as _f:
-        content = yaml.safe_load(_f)
-        content["disable_hooks_for_plugins"] = [plugin]
-        yaml.safe_dump(content, _f)
+    """Set `disable_hooks_for_plugins` in `settings.py`."""
+    settings_path = (
+        context.root_project_dir / "src" / context.package_name / "settings.py"
+    )
+    to_add = f"""\nDISABLE_HOOKS_FOR_PLUGINS = ("{plugin}",)"""
+    with settings_path.open("a") as settings_file:
+        settings_file.write(to_add)
 
 
 @given("I have initialized a git repository")
@@ -305,6 +306,7 @@ def create_project_with_starter(context):
             str(starter_dir),
         ],
         env=context.env,
+        cwd=context.temp_dir,
     )
     assert res.returncode == OK_EXIT_CODE, res
 
@@ -314,7 +316,11 @@ def create_project_with_starter(context):
 def create_project_without_starter(context):
     """Behave step to run kedro new given the config I previously created.
     """
-    res = run([context.kedro, "new", "-c", str(context.config_file)], env=context.env)
+    res = run(
+        [context.kedro, "new", "-c", str(context.config_file)],
+        env=context.env,
+        cwd=context.temp_dir,
+    )
     assert res.returncode == OK_EXIT_CODE, res
 
 
@@ -379,7 +385,7 @@ def exec_kedro_run_with_tag(context, cmd, tags):
 @when("I ask the CLI for a version")
 def get_kedro_version(context):
     """Behave step to run `kedro -V`."""
-    res = run([context.kedro, "-V"], env=context.env)
+    res = run([context.kedro, "-V"], env=context.env, cwd=context.temp_dir)
     context.version_str = res.stdout
     assert context.version_str, res  # check non-empty
 
@@ -388,7 +394,7 @@ def get_kedro_version(context):
 def get_kedro_version_python(context):
     """Behave step to run `python -m kedro -V`."""
     cmd = [context.python, "-m", "kedro", "-V"]
-    context.version_str = run(cmd, env=context.env).stdout
+    context.version_str = run(cmd, env=context.env, cwd=context.temp_dir).stdout
     assert context.version_str  # check non-empty
 
 
@@ -438,26 +444,22 @@ def do_git_reset_hard(context):
 
 @when('I move the package to "{new_source_dir}"')
 def move_package(context: behave.runner.Context, new_source_dir):
-    """Move the project package to a new directory.
-    """
+    """Move the project package to a new directory."""
     current_src_path = (context.root_project_dir / "src").resolve()
     new_src_path = (context.root_project_dir / new_source_dir).resolve()
 
     new_src_path.mkdir(exist_ok=True)
     shutil.move(str(current_src_path / context.package_name), str(new_src_path))
 
 
-@when('Source directory is updated to "{new_source_dir}" in kedro.yml')
-def update_kedro_yml(context: behave.runner.Context, new_source_dir):
-    """Update `source_dir` in .kedro.yml file.
-    """
-
-    kedro_yml_path = context.root_project_dir / ".kedro.yml"
-
-    with kedro_yml_path.open("r+") as _f:
-        content = yaml.safe_load(_f)
-        content["source_dir"] = new_source_dir
-        yaml.safe_dump(content, _f)
+@when('Source directory is updated to "{new_source_dir}" in pyproject.toml')
+def update_pyproject_toml(context: behave.runner.Context, new_source_dir):
+    """Update `source_dir` in pyproject.toml file."""
+    pyproject_toml_path = context.root_project_dir / "pyproject.toml"
+    content = toml.load(pyproject_toml_path)
+    content["tool"]["kedro"]["source_dir"] = new_source_dir
+    content_str = toml.dumps(content)
+    pyproject_toml_path.write_text(content_str)
 
 
 @given("I have updated kedro requirements")

features/steps/hooks_template.py

@@ -93,6 +93,3 @@ def register_catalog(  # pylint: disable=no-self-use, too-many-arguments
         return DataCatalog.from_config(
             catalog, credentials, load_versions, save_version, journal
         )
-
-
-project_hooks = ProjectHooks()

features/steps/pipeline_steps.py

@@ -163,7 +163,11 @@ def _create_template_project(context):
     with context.config_file.open("w") as config_file:
         yaml.dump(config, config_file, default_flow_style=False)
 
-    res = run([context.kedro, "new", "-c", str(context.config_file)], env=context.env)
+    res = run(
+        [context.kedro, "new", "-c", str(context.config_file)],
+        env=context.env,
+        cwd=str(context.temp_dir),
+    )
     assert res.returncode == 0
 
     _setup_template_files(context)
@@ -204,18 +208,18 @@ def create_template_with_pipeline(context):
 
 @given('I have defined a node "{node_name}" tagged with {tags:CSV}')
 def node_tagged_with(context, node_name, tags):
-    """
-    Check tagging in `hooks_template.py` is consistent with tagging
+    """Check tagging in `hooks_template.py` is consistent with tagging
     descriptions in background steps
     """
     sys.path.append(
         str(context.root_project_dir / "src" / context.project_name.replace("-", "_"))
     )
     # pylint: disable=import-outside-toplevel
-    import hooks
+    from hooks import ProjectHooks
 
     # pylint: disable=no-member
-    context.project_pipeline = hooks.project_hooks.register_pipelines()["__default__"]
+    context.project_pipeline = ProjectHooks().register_pipelines()["__default__"]
+
     node_objs = [n for n in context.project_pipeline.nodes if n.name == node_name]
     assert node_objs
     assert set(tags) == node_objs[0].tags

features/steps/test_starter/{{ cookiecutter.repo_name }}/pyproject.toml

@@ -0,0 +1,5 @@
+[tool.kedro]
+context_path = "{{ cookiecutter.python_package }}.run.ProjectContext"
+project_name = "{{ cookiecutter.project_name }}"
+project_version = "{{ cookiecutter.kedro_version }}"
+package_name = "{{ cookiecutter.python_package }}"

features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py

@@ -73,6 +73,3 @@ def register_catalog(
         return DataCatalog.from_config(
             catalog, credentials, load_versions, save_version, journal
         )
-
-
-project_hooks = ProjectHooks()

features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py

@@ -0,0 +1,42 @@
+# Copyright 2020 QuantumBlack Visual Analytics Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS
+# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo
+# (either separately or in combination, "QuantumBlack Trademarks") are
+# trademarks of QuantumBlack. The License does not grant you any right or
+# license to the QuantumBlack Trademarks. You may not use the QuantumBlack
+# Trademarks or any confusingly similar mark as a trademark for your product,
+# or use the QuantumBlack Trademarks in any other manner that might cause
+# confusion in the marketplace, including but not limited to in advertising,
+# on websites, or on software.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Project settings."""
+from {{cookiecutter.python_package}}.hooks import ProjectHooks
+
+# Instantiate and list your project hooks here
+HOOKS = (ProjectHooks(),)
+
+# List the installed plugins for which to disable auto-registry
+# DISABLE_HOOKS_FOR_PLUGINS = ("kedro-viz",)
+
+# Define where to store data from a KedroSession
+# SESSION_STORE = {
+#     "type": "kedro.framework.session.store.ShelveStore",
+#     "path": "."
+# }

features/windows_reqs.txt

@@ -8,5 +8,6 @@ pandas>=0.24.0, <1.0.4
 pluggy~=0.13.0
 psutil==5.6.6
 requests~=2.20
+toml~=0.10.1
 toposort~=1.5
 PyYAML>=4.2, <6.0

kedro/framework/cli/cli.py

@@ -633,9 +633,9 @@ def main():  # pragma: no cover
     project_groups = []
 
     path = Path.cwd()
-    kedro_yaml_path = path / ".kedro.yml"
+    pyproject_toml = path / "pyproject.toml"
 
-    if kedro_yaml_path.exists():
+    if pyproject_toml.is_file():
         # load project commands from cli.py
         static_data = get_static_project_data(path)
         source_dir = static_data["source_dir"]