Merge branch 'main' into dataset-api-add_save_method

.circleci/continue_config.yml

@@ -69,6 +69,7 @@ commands:
           command: |
             cd <<parameters.plugin>>
             pip install git+https://github.com/kedro-org/kedro@main
+            pip install .
             pip install -r test_requirements.txt
       - run:
           name: Install pre-commit hooks

.github/workflows/check-plugin.yml

@@ -0,0 +1,134 @@
+name: Running tests and linter
+
+on:
+  workflow_call:
+    inputs:
+      plugin:
+        type: string
+
+jobs:
+  unit-tests:
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{matrix.python-version}}
+      - name: Cache python packages for Linux
+        if: matrix.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Cache python packages for Windows
+        if: matrix.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install Kedro
+        run: pip install git+https://github.com/kedro-org/kedro@main
+      - name: Install dependencies
+        run: |
+          cd ${{ inputs.plugin }}
+          pip install -r test_requirements.txt
+      - name: Install pytables (only for kedro-datasets on windows)
+        if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets'
+        run: pip install tables
+      - name: pip freeze
+        run: pip freeze
+      - name: Run unit tests for Linux / all plugins
+        if: matrix.os != 'windows-latest'
+        run: make plugin=${{ inputs.plugin }} test
+      - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry
+        if: matrix.os == 'windows-latest' && inputs.plugin != 'kedro-datasets'
+        run: |
+          cd ${{ inputs.plugin }}
+          pytest tests
+      - name: Run unit tests for Windows / kedro-datasets / no spark sequential
+        if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10'
+        run: |
+          make test-no-spark-sequential
+      - name: Run unit tests for Windows / kedro-datasets / no spark parallel
+        if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version != '3.10'
+        run: |
+          make test-no-spark
+
+  lint:
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install dependencies
+        run: |
+            cd ${{ inputs.plugin }}
+            pip install git+https://github.com/kedro-org/kedro@main
+            pip install -r test_requirements.txt
+            pip freeze
+      - name: Install pre-commit hooks
+        run: |
+            cd ${{ inputs.plugin }}
+            pre-commit install --install-hooks
+            pre-commit install --hook-type pre-push
+      - name: Run linter
+        run: make plugin=${{ inputs.plugin }} lint
+
+  e2e-tests:
+    if: inputs.plugin != 'kedro-datasets'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{matrix.python-version}}
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install dependencies
+        run: |
+          cd ${{ inputs.plugin }}
+          pip install git+https://github.com/kedro-org/kedro@main
+          pip install -r test_requirements.txt
+      - name: pip freeze
+        run: pip freeze
+      - name: Run end to end tests
+        # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run`
+        # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241
+        shell: 'script -q -e -c "bash {0}"'
+        run: make plugin=${{ inputs.plugin }} e2e-tests

.github/workflows/kedro-airflow.yml

@@ -0,0 +1,16 @@
+name: Run checks on kedro-airflow
+
+on:
+  push:
+    paths:
+      - "kedro-airflow/**"
+  pull_request:
+    paths:
+      - "kedro-airflow/**"
+    types: [ synchronize ]
+
+jobs:
+  airflow-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-airflow

.github/workflows/kedro-datasets.yml

@@ -0,0 +1,16 @@
+name: Run checks on kedro-datasets
+
+on:
+  push:
+    paths:
+      - "kedro-datasets/**"
+  pull_request:
+    paths:
+      - "kedro-datasets/**"
+    types: [ synchronize ]
+
+jobs:
+  datasets-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-datasets

.github/workflows/kedro-docker.yml

@@ -0,0 +1,16 @@
+name: Run checks on kedro-docker
+
+on:
+  push:
+    paths:
+      - "kedro-docker/**"
+  pull_request:
+    paths:
+      - "kedro-docker/**"
+    types: [ synchronize ]
+
+jobs:
+  docker-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-docker

.github/workflows/kedro-telemetry.yml

@@ -0,0 +1,16 @@
+name: Run checks on kedro-telemetry
+
+on:
+  push:
+    paths:
+      - "kedro-telemetry/**"
+  pull_request:
+    paths:
+      - "kedro-telemetry/**"
+    types: [ synchronize ]
+
+jobs:
+  telemetry-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-telemetry

.gitpod.yml

@@ -0,0 +1,33 @@
+# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
+image: gitpod/workspace-python-3.10:2023-04-20-16-32-37
+
+
+tasks:
+  # We want packages installed during the pre-build init steps to go to /workspace
+  # rather than ~ so that they are persisted. Gitpod sets PIP_USER=yes to ensure this,
+  # but pre-commit requires PIP_USER=no. Hence we set PIP_USER=no and use
+  # pip install --user to install to /workspace.
+  - name: kedro-plugins
+    before: |
+      echo PIP_USER=no >> ~/.bashrc && export PIP_USER=no
+    init: |
+      make sign-off
+    command: |
+      pre-commit install --install-hooks
+      clear
+
+
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: true
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: false
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: true

kedro-airflow/MANIFEST.in

@@ -1,4 +1,3 @@
 include README.md
 include LICENSE.md
-include requirements.txt
 include kedro_airflow/airflow_dag_template.j2

kedro-airflow/RELEASE.md

@@ -1,5 +1,6 @@
 # Upcoming release 0.5.2
 * Change reference to `kedro.pipeline.Pipeline` object throughout test suite with `kedro.modular_pipeline.pipeline` factory.
+* Migrate all project metadata to static `pyproject.toml`.
 
 # Release 0.5.1
 * Added additional CLI argument `--jinja-file` to provide a path to a custom Jinja2 template.

kedro-airflow/pyproject.toml

@@ -1,3 +1,51 @@
+[build-system]
+requires = ["setuptools>=61.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kedro-airflow"
+authors = [
+    {name = "Kedro"}
+]
+description = "Kedro-Airflow makes it easy to deploy Kedro projects to Airflow"
+requires-python = ">=3.7, <3.11"
+license = {text = "Apache Software License (Apache 2.0)"}
+dependencies = [
+    "kedro>=0.17.5",
+    "python-slugify>=4.0",
+    "semver~=2.10",  # Needs to be at least 2.10.0 to get VersionInfo.match
+]
+dynamic = ["readme", "version"]
+
+[project.urls]
+Source = "https://github.com/kedro-org/kedro-plugins/tree/main/kedro-airflow"
+Documentation = "https://github.com/kedro-org/kedro-plugins/blob/main/kedro-airflow/README.md"
+Tracker = "https://github.com/kedro-org/kedro-plugins/issues"
+
+[project.entry-points."kedro.project_commands"]
+airflow = "kedro_airflow.plugin:commands"
+
+[tool.setuptools]
+include-package-data = true
+packages = ["kedro_airflow"]
+zip-safe = false
+
+[tool.setuptools.package-data]
+kedro_airflow = ["kedro_airflow/airflow_dag_template.j2"]
+
+[tool.setuptools.dynamic]
+readme = {file = "README.md", content-type = "text/markdown"}
+version = {attr = "kedro_airflow.__version__"}
+
+[tool.pytest.ini_options]
+addopts = """
+    --cov-report xml:coverage.xml
+    --cov-report term-missing
+    --cov kedro_airflow
+    --cov tests
+    --no-cov-on-fail
+    -ra"""
+
 [tool.black]
 exclude=".*template.py"
 

kedro-airflow/test_requirements.txt

@@ -1,4 +1,3 @@
--r requirements.txt
 apache-airflow<3.0
 bandit>=1.6.2, <2.0
 behave

kedro-datasets/pyproject.toml

@@ -1,3 +1,7 @@
+[build-system]
+requires = ["setuptools>=61.2"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "kedro-datasets"
 authors = [