Setup CircleCI builds for all repos in kedro-plugins

.circleci/config.yml

@@ -1,4 +1,223 @@
-version: 2
+version: 2.1
+
+orbs:
+  win: circleci/windows@2.4.0
+
+commands:
+  setup_conda:
+    parameters:
+      python_version:
+        type: string
+    steps:
+      - run:
+          name: Create virtual env
+          command: |
+            # Get rid of pyenv stuff
+            sudo rm -rf .pyenv/ /opt/circleci/.pyenv/
+            # Download and install miniconda
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh  > miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            # Create an anaconda virtualenv for python_version and make that the default python interpreter
+            echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV
+            echo "conda deactivate; conda activate kedro_plugins" >> $BASH_ENV
+            . /home/circleci/miniconda/etc/profile.d/conda.sh
+            conda create --name kedro_plugins python=<<parameters.python_version>> -y
+            source $BASH_ENV
+
+  setup_requirements:
+    parameters:
+      plugin:
+        type: string
+    steps:
+      - run:
+          name: Install pip setuptools
+          command: |
+            make install-pip-setuptools
+      - run:
+          name: Install kedro and test requirements
+          command: |
+            cd <<parameters.plugin>>
+            pip install git+https://github.com/kedro-org/kedro
+            pip install -r test_requirements.txt
+      - run:
+          name: Install pre-commit hooks
+          command: |
+            cd <<parameters.plugin>>
+            pre-commit install --install-hooks
+            pre-commit install --hook-type pre-push
+
+  setup:
+    parameters:
+      python_version:
+        type: string
+      plugin:
+        type: string
+    steps:
+      - checkout
+      - setup_conda:
+          python_version: <<parameters.python_version>>
+      - setup_requirements:
+          plugin: <<parameters.plugin>>
+
+  # Windows specific commands
+  win_setup_conda:
+    # Miniconda3 is pre-installed on the machine:
+    # https://circleci.com/docs/2.0/hello-world-windows
+    description: Setup conda
+    parameters:
+      python_version:
+        type: string
+    steps:
+      - run:
+          name: Initialize conda
+          command: conda init powershell
+      - run:
+          name: Create 'kedro_plugins' conda environment
+          command: |
+            conda create --name kedro_plugins python=<<parameters.python_version>> -y
+
+  win_setup_requirements:
+    description: Install Kedro plugins dependencies
+    parameters:
+      plugin:
+        type: string
+    steps:
+      - run:
+          name: Install Kedro plugins dependencies
+          command: |
+            conda activate kedro_plugins
+            cd <<parameters.plugin>>
+            python -m pip install -U pip setuptools wheel
+            pip install git+https://github.com/kedro-org/kedro@main
+            pip install -r test_requirements.txt -U
+
+  win_tests:
+    description: Run build on Windows
+    parameters:
+      python_version:
+        type: string
+      plugin:
+        type: string
+    steps:
+      - checkout
+      - win_setup_conda:
+          python_version: <<parameters.python_version>>
+      - win_setup_requirements:
+          plugin: <<parameters.plugin>>
+      - run:
+          # e2e tests are not currently runnable on CircleCI on Windows as
+          # those require the ability to run Linux containers:
+          # "The Windows executor currently only supports Windows containers.
+          # Running Linux containers on Windows is not possible for now"
+          # (from https://circleci.com/docs/2.0/hello-world-windows/)
+          name: Run unit tests
+          command: |
+            conda activate kedro_plugins
+            pytest <<parameters.plugin>>\tests
+
+jobs:
+  unit_tests:
+    parameters:
+      python_version:
+        type: string
+      plugin:
+        type: string
+    machine:
+      # Don't use 2018 image: https://discuss.circleci.com/t/24639/18
+      image: circleci/classic:201711-01
+      docker_layer_caching: true
+    environment:
+      python_version: <<parameters.python_version>>
+    steps:
+      - setup:
+          python_version: <<parameters.python_version>>
+          plugin: <<parameters.plugin>>
+      - run:
+          name: Run tests
+          command: |
+            make plugin=<<parameters.plugin>> test
+
+  e2e_tests:
+    parameters:
+      python_version:
+        type: string
+      plugin:
+        type: string
+    machine:
+      # Don't use 2018 image: https://discuss.circleci.com/t/24639/18
+      image: circleci/classic:201711-01
+      docker_layer_caching: true
+    environment:
+      python_version: <<parameters.python_version>>
+    steps:
+      - setup:
+          python_version: <<parameters.python_version>>
+          plugin: <<parameters.plugin>>
+      - run:
+          name: Run all end to end tests
+          command: |
+            make plugin=<<parameters.plugin>> e2e-tests
+
+  lint:
+    parameters:
+      python_version:
+        type: string
+      plugin:
+        type: string
+    machine:
+      # Don't use 2018 image: https://discuss.circleci.com/t/24639/18
+      image: circleci/classic:201711-01
+      docker_layer_caching: true
+    environment:
+      python_version: <<parameters.python_version>>
+    steps:
+      - setup:
+          python_version: <<parameters.python_version>>
+          plugin: <<parameters.plugin>>
+      - run:
+          name: Run pylint and flake8
+          command: |
+            make plugin=<<parameters.plugin>> lint
+
+  # Windows jobs
+  win_build:
+    parameters:
+      python_version:
+        type: string
+      plugin:
+        type: string
+    executor:
+      name: win/default
+    working_directory: ~/repo
+    environment:
+      python_version: <<parameters.python_version>>
+    steps:
+      - win_tests:
+          python_version: <<parameters.python_version>>
+          plugin: <<parameters.plugin>>
+
 
 workflows:
   version: 2
+  regular:
+    jobs:
+      - unit_tests:
+          matrix:
+            parameters:
+              python_version: [ "3.6", "3.7", "3.8" ]
+              plugin: [ "kedro-telemetry", "kedro-docker", "kedro-airflow" ]
+      - e2e_tests:
+          matrix:
+            parameters:
+              python_version: [ "3.6", "3.7", "3.8" ]
+              plugin: [ "kedro-docker", "kedro-airflow" ]
+      - win_build:
+          matrix:
+            parameters:
+              python_version: [ "3.6", "3.7", "3.8" ]
+              plugin: [ "kedro-telemetry", "kedro-docker", "kedro-airflow" ]
+      - lint:
+          matrix:
+            parameters:
+              python_version: [ "3.8" ]
+              plugin: [ "kedro-telemetry", "kedro-docker", "kedro-airflow" ]

kedro-telemetry/Makefile → Makefile

@@ -9,10 +9,13 @@ install-pip-setuptools:
 	python -m pip install -U pip setuptools wheel
 
 lint:
-	pre-commit run -a --hook-stage manual
+	cd $(plugin) && pre-commit run -a --hook-stage manual
 
 test:
-	pytest -vv tests
+	pytest -vv $(plugin)/tests
+
+e2e-tests:
+	cd $(plugin) && behave
 
 secret-scan:
 	trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt .

kedro-airflow/.pre-commit-config.yaml

@@ -17,11 +17,15 @@ repos:
         exclude: "^kedro_airflow/dag_template.py"
       - id: requirements-txt-fixer # Sorts entries in requirements.txt
       - id: flake8
+        files: ^kedro-airflow/kedro_airflow/
+        args:
+          - "--max-line-length=88"
+          - "--max-complexity=18"
+          - "--max-complexity=18"
+          - "--select=B,C,E,F,W,T4,B9"
+          - "--ignore=E203,E266,E501,W503"
+        exclude: "^template.py"
 
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
-    hooks:
-      - id: isort
 
   - repo: local
     hooks:
@@ -32,21 +36,21 @@ repos:
         name: "Quick PyLint on kedro_airflow/*"
         language: system
         types: [file, python]
-        files: ^kedro_airflow/
+        files: ^kedro-airflow/kedro_airflow/
         entry: pylint --disable=unnecessary-pass
         stages: [commit]
       - id: pylint-quick-features
         name: "Quick PyLint on features/*"
         language: system
         types: [file, python]
-        files: ^features/
+        files: ^kedro-airflow/features/
         entry: pylint --disable=missing-docstring,no-name-in-module
         stages: [commit]
       - id: pylint-quick-tests
         name: "Quick PyLint on tests/*"
         language: system
         types: [file, python]
-        files: ^tests/
+        files: ^kedro-airflow/tests/
         entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments
         stages: [commit]
       # The same pylint checks, but running on all files. It's for manual run with `make lint`
@@ -55,24 +59,30 @@ repos:
         language: system
         pass_filenames: false
         stages: [manual]
-        entry: pylint --disable=unnecessary-pass kedro_airflow
+        entry: pylint --disable=unnecessary-pass kedro-airflow/kedro_airflow
       - id: pylint-features
         name: "PyLint on features/*"
         language: system
         pass_filenames: false
         stages: [manual]
-        entry: pylint --disable=missing-docstring,no-name-in-module features
+        entry: pylint --disable=missing-docstring,no-name-in-module kedro-airflow/features
       - id: pylint-tests
         name: "PyLint on tests/*"
         language: system
         pass_filenames: false
         stages: [manual]
-        entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments tests
+        entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-airflow/tests
+      - id: isort
+        name: "Sort imports"
+        language: system
+        types: [ file, python ]
+        files: ^kedro-airflow/
+        entry: isort
       - id: black
         name: "Black"
         language: system
         pass_filenames: false
-        entry: black kedro_airflow features tests
+        entry: black kedro-airflow/kedro_airflow kedro-airflow/features kedro-airflow/tests
       - id: secret_scan
         name: "Secret scan"
         language: system
@@ -82,5 +92,5 @@ repos:
         name: "Bandit security check"
         language: system
         types: [file, python]
-        exclude: ^tests/|^features/
+        exclude: ^kedro-airflow/tests/|^kedro-airflow/features/|^kedro-telemetry|^kedro-docker/
         entry: bandit -ll

kedro-airflow/features/steps/cli_steps.py

@@ -2,7 +2,6 @@
 
 import yaml
 from behave import given, then, when
-
 from features.steps.sh_run import run
 
 OK_EXIT_CODE = 0

kedro-airflow/tests/test_plugin.py

@@ -2,7 +2,6 @@
 
 from kedro.framework.project import pipelines
 from kedro.pipeline import Pipeline, node
-
 from kedro_airflow.plugin import commands