diff --git a/.circleci/config.yml b/.circleci/config.yml index a0ca7bf65..23a728704 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,223 @@ -version: 2 +version: 2.1 + +orbs: + win: circleci/windows@2.4.0 + +commands: + setup_conda: + parameters: + python_version: + type: string + steps: + - run: + name: Create virtual env + command: | + # Get rid of pyenv stuff + sudo rm -rf .pyenv/ /opt/circleci/.pyenv/ + # Download and install miniconda + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + # Create an anaconda virtualenv for python_version and make that the default python interpreter + echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV + echo "conda deactivate; conda activate kedro_plugins" >> $BASH_ENV + . /home/circleci/miniconda/etc/profile.d/conda.sh + conda create --name kedro_plugins python=<> -y + source $BASH_ENV + + setup_requirements: + parameters: + plugin: + type: string + steps: + - run: + name: Install pip setuptools + command: | + make install-pip-setuptools + - run: + name: Install kedro and test requirements + command: | + cd <> + pip install git+https://github.com/kedro-org/kedro + pip install -r test_requirements.txt + - run: + name: Install pre-commit hooks + command: | + cd <> + pre-commit install --install-hooks + pre-commit install --hook-type pre-push + + setup: + parameters: + python_version: + type: string + plugin: + type: string + steps: + - checkout + - setup_conda: + python_version: <> + - setup_requirements: + plugin: <> + + # Windows specific commands + win_setup_conda: + # Miniconda3 is pre-installed on the machine: + # https://circleci.com/docs/2.0/hello-world-windows + description: Setup conda + parameters: + python_version: + type: string + steps: + - run: + name: Initialize conda + command: conda init powershell + - run: + name: Create 'kedro_plugins' conda environment + command: | + conda create --name kedro_plugins python=<> -y + + win_setup_requirements: + description: Install Kedro plugins dependencies + parameters: + plugin: + type: string + steps: + - run: + name: Install Kedro plugins dependencies + command: | + conda activate kedro_plugins + cd <> + python -m pip install -U pip setuptools wheel + pip install git+https://github.com/kedro-org/kedro@main + pip install -r test_requirements.txt -U + + win_tests: + description: Run build on Windows + parameters: + python_version: + type: string + plugin: + type: string + steps: + - checkout + - win_setup_conda: + python_version: <> + - win_setup_requirements: + plugin: <> + - run: + # e2e tests are not currently runnable on CircleCI on Windows as + # those require the ability to run Linux containers: + # "The Windows executor currently only supports Windows containers. + # Running Linux containers on Windows is not possible for now" + # (from https://circleci.com/docs/2.0/hello-world-windows/) + name: Run unit tests + command: | + conda activate kedro_plugins + pytest <>\tests + +jobs: + unit_tests: + parameters: + python_version: + type: string + plugin: + type: string + machine: + # Don't use 2018 image: https://discuss.circleci.com/t/24639/18 + image: circleci/classic:201711-01 + docker_layer_caching: true + environment: + python_version: <> + steps: + - setup: + python_version: <> + plugin: <> + - run: + name: Run tests + command: | + make plugin=<> test + + e2e_tests: + parameters: + python_version: + type: string + plugin: + type: string + machine: + # Don't use 2018 image: https://discuss.circleci.com/t/24639/18 + image: circleci/classic:201711-01 + docker_layer_caching: true + environment: + python_version: <> + steps: + - setup: + python_version: <> + plugin: <> + - run: + name: Run all end to end tests + command: | + make plugin=<> e2e-tests + + lint: + parameters: + python_version: + type: string + plugin: + type: string + machine: + # Don't use 2018 image: https://discuss.circleci.com/t/24639/18 + image: circleci/classic:201711-01 + docker_layer_caching: true + environment: + python_version: <> + steps: + - setup: + python_version: <> + plugin: <> + - run: + name: Run pylint and flake8 + command: | + make plugin=<> lint + + # Windows jobs + win_build: + parameters: + python_version: + type: string + plugin: + type: string + executor: + name: win/default + working_directory: ~/repo + environment: + python_version: <> + steps: + - win_tests: + python_version: <> + plugin: <> + workflows: version: 2 + regular: + jobs: + - unit_tests: + matrix: + parameters: + python_version: [ "3.6", "3.7", "3.8" ] + plugin: [ "kedro-telemetry", "kedro-docker", "kedro-airflow" ] + - e2e_tests: + matrix: + parameters: + python_version: [ "3.6", "3.7", "3.8" ] + plugin: [ "kedro-docker", "kedro-airflow" ] + - win_build: + matrix: + parameters: + python_version: [ "3.6", "3.7", "3.8" ] + plugin: [ "kedro-telemetry", "kedro-docker", "kedro-airflow" ] + - lint: + matrix: + parameters: + python_version: [ "3.8" ] + plugin: [ "kedro-telemetry", "kedro-docker", "kedro-airflow" ] diff --git a/kedro-telemetry/Makefile b/Makefile similarity index 84% rename from kedro-telemetry/Makefile rename to Makefile index da455ec2d..2ef6b7f6a 100644 --- a/kedro-telemetry/Makefile +++ b/Makefile @@ -9,10 +9,13 @@ install-pip-setuptools: python -m pip install -U pip setuptools wheel lint: - pre-commit run -a --hook-stage manual + cd $(plugin) && pre-commit run -a --hook-stage manual test: - pytest -vv tests + pytest -vv $(plugin)/tests + +e2e-tests: + cd $(plugin) && behave secret-scan: trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt . diff --git a/kedro-airflow/.flake8 b/kedro-airflow/.flake8 deleted file mode 100644 index a3a72c4e6..000000000 --- a/kedro-airflow/.flake8 +++ /dev/null @@ -1,8 +0,0 @@ -# Copied from Black - -[flake8] -ignore = E203, E266, E501, W503 -max-line-length = 80 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 -exclude = *template.py diff --git a/kedro-airflow/.isort.cfg b/kedro-airflow/.isort.cfg deleted file mode 100644 index 5117abefb..000000000 --- a/kedro-airflow/.isort.cfg +++ /dev/null @@ -1,10 +0,0 @@ -# copied from black - -[settings] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -known_first_party=features,kedro_airflow,tests -default_section=THIRDPARTY diff --git a/kedro-airflow/.pre-commit-config.yaml b/kedro-airflow/.pre-commit-config.yaml index afbb9fa24..bbf5f45c0 100644 --- a/kedro-airflow/.pre-commit-config.yaml +++ b/kedro-airflow/.pre-commit-config.yaml @@ -17,11 +17,15 @@ repos: exclude: "^kedro_airflow/dag_template.py" - id: requirements-txt-fixer # Sorts entries in requirements.txt - id: flake8 + files: ^kedro-airflow/kedro_airflow/ + args: + - "--max-line-length=88" + - "--max-complexity=18" + - "--max-complexity=18" + - "--select=B,C,E,F,W,T4,B9" + - "--ignore=E203,E266,E501,W503" + exclude: "^template.py" - - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.21 - hooks: - - id: isort - repo: local hooks: @@ -32,21 +36,21 @@ repos: name: "Quick PyLint on kedro_airflow/*" language: system types: [file, python] - files: ^kedro_airflow/ + files: ^kedro-airflow/kedro_airflow/ entry: pylint --disable=unnecessary-pass stages: [commit] - id: pylint-quick-features name: "Quick PyLint on features/*" language: system types: [file, python] - files: ^features/ + files: ^kedro-airflow/features/ entry: pylint --disable=missing-docstring,no-name-in-module stages: [commit] - id: pylint-quick-tests name: "Quick PyLint on tests/*" language: system types: [file, python] - files: ^tests/ + files: ^kedro-airflow/tests/ entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments stages: [commit] # The same pylint checks, but running on all files. It's for manual run with `make lint` @@ -55,24 +59,30 @@ repos: language: system pass_filenames: false stages: [manual] - entry: pylint --disable=unnecessary-pass kedro_airflow + entry: pylint --disable=unnecessary-pass kedro-airflow/kedro_airflow - id: pylint-features name: "PyLint on features/*" language: system pass_filenames: false stages: [manual] - entry: pylint --disable=missing-docstring,no-name-in-module features + entry: pylint --disable=missing-docstring,no-name-in-module kedro-airflow/features - id: pylint-tests name: "PyLint on tests/*" language: system pass_filenames: false stages: [manual] - entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments tests + entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-airflow/tests + - id: isort + name: "Sort imports" + language: system + types: [ file, python ] + files: ^kedro-airflow/ + entry: isort - id: black name: "Black" language: system pass_filenames: false - entry: black kedro_airflow features tests + entry: black kedro-airflow/kedro_airflow kedro-airflow/features kedro-airflow/tests - id: secret_scan name: "Secret scan" language: system @@ -82,5 +92,5 @@ repos: name: "Bandit security check" language: system types: [file, python] - exclude: ^tests/|^features/ + exclude: ^kedro-airflow/tests/|^kedro-airflow/features/|^kedro-telemetry|^kedro-docker/ entry: bandit -ll diff --git a/kedro-airflow/Makefile b/kedro-airflow/Makefile deleted file mode 100644 index 4ef447e80..000000000 --- a/kedro-airflow/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -package: - rm -Rf dist - python setup.py sdist bdist_wheel - -install: package - pip install -U dist/*.whl - -install-pip-setuptools: - python -m pip install -U pip setuptools wheel - -lint: - pre-commit run -a --hook-stage manual - -test: - pytest -vv tests - -e2e-tests: - behave - -secret-scan: - trufflehog --max_depth 1 . - -clean: - rm -rf build dist pip-wheel-metadata .pytest_cache - find . -regex ".*/__pycache__" -exec rm -rf {} + - find . -regex ".*\.egg-info" -exec rm -rf {} + diff --git a/kedro-airflow/features/steps/cli_steps.py b/kedro-airflow/features/steps/cli_steps.py index 19571c070..9fbde1df2 100644 --- a/kedro-airflow/features/steps/cli_steps.py +++ b/kedro-airflow/features/steps/cli_steps.py @@ -2,7 +2,6 @@ import yaml from behave import given, then, when - from features.steps.sh_run import run OK_EXIT_CODE = 0 diff --git a/kedro-airflow/tests/test_plugin.py b/kedro-airflow/tests/test_plugin.py index 53c000b30..971be623c 100644 --- a/kedro-airflow/tests/test_plugin.py +++ b/kedro-airflow/tests/test_plugin.py @@ -2,7 +2,6 @@ from kedro.framework.project import pipelines from kedro.pipeline import Pipeline, node - from kedro_airflow.plugin import commands diff --git a/kedro-docker/.flake8 b/kedro-docker/.flake8 deleted file mode 100644 index 4fff5484f..000000000 --- a/kedro-docker/.flake8 +++ /dev/null @@ -1,7 +0,0 @@ -# copied from black - -[flake8] -ignore = E203, E266, E501, W503 -max-line-length = 80 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 diff --git a/kedro-docker/.isort.cfg b/kedro-docker/.isort.cfg deleted file mode 100644 index 06370f898..000000000 --- a/kedro-docker/.isort.cfg +++ /dev/null @@ -1,10 +0,0 @@ -# copied from black - -[settings] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -known_first_party=features,kedro_docker,tests -default_section=THIRDPARTY diff --git a/kedro-docker/.pre-commit-config.yaml b/kedro-docker/.pre-commit-config.yaml index 957134eb7..fb3de1c5b 100644 --- a/kedro-docker/.pre-commit-config.yaml +++ b/kedro-docker/.pre-commit-config.yaml @@ -16,11 +16,13 @@ repos: - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source. - id: requirements-txt-fixer # Sorts entries in requirements.txt - id: flake8 - - - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.21 - hooks: - - id: isort + files: ^kedro-docker/kedro_docker/ + args: + - "--max-line-length=88" + - "--max-complexity=18" + - "--max-complexity=18" + - "--select=B,C,E,F,W,T4,B9" + - "--ignore=E203,E266,E501,W503" - repo: local hooks: @@ -31,21 +33,21 @@ repos: name: "Quick PyLint on kedro_docker/*" language: system types: [file, python] - files: ^kedro_docker/ + files: ^kedro-docker/kedro_docker/ entry: pylint --disable=unnecessary-pass stages: [commit] - id: pylint-quick-features name: "Quick PyLint on features/*" language: system types: [file, python] - files: ^features/ + files: ^kedro-docker/features/ entry: pylint --disable=missing-docstring,no-name-in-module stages: [commit] - id: pylint-quick-tests name: "Quick PyLint on tests/*" language: system types: [file, python] - files: ^tests/ + files: ^kedro-docker/tests/ entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments stages: [commit] # The same pylint checks, but running on all files. It's for manual run with `make lint` @@ -54,32 +56,38 @@ repos: language: system pass_filenames: false stages: [manual] - entry: pylint --disable=unnecessary-pass kedro_docker + entry: pylint --disable=unnecessary-pass kedro-docker/kedro_docker - id: pylint-features name: "PyLint on features/*" language: system pass_filenames: false stages: [manual] - entry: pylint --disable=missing-docstring,no-name-in-module features + entry: pylint --disable=missing-docstring,no-name-in-module kedro-docker/features - id: pylint-tests name: "PyLint on tests/*" language: system pass_filenames: false stages: [manual] - entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments tests + entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-docker/tests + - id: isort + name: "Sort imports" + language: system + types: [ file, python ] + files: ^kedro-docker/ + entry: isort - id: black name: "Black" language: system pass_filenames: false - entry: black kedro_docker features tests + entry: black kedro-docker/kedro_docker kedro-docker/features kedro-docker/tests - id: secret_scan name: "Secret scan" language: system pass_filenames: false - entry: make secret-scan + entry: make plugin=kedro-docker secret-scan - id: bandit name: "Bandit security check" language: system types: [file, python] - exclude: ^tests/|^features/ + exclude: ^kedro-docker/tests/|^kedro-docker/features/|^kedro-telemetry|^kedro-airflow entry: bandit -ll diff --git a/kedro-docker/Makefile b/kedro-docker/Makefile deleted file mode 100644 index b523889ef..000000000 --- a/kedro-docker/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -clean: - rm -rf build dist pip-wheel-metadata .pytest_cache - find . -regex ".*/__pycache__" -exec rm -rf {} + - find . -regex ".*\.egg-info" -exec rm -rf {} + - -install: - pip install . - -install-pip-setuptools: - python -m pip install -U "pip>=20.0" "setuptools>=38.0" wheel - -lint: - pre-commit run -a --hook-stage manual - -test: - pytest tests - -e2e-tests: - behave - -secret-scan: - trufflehog --max_depth 1 . - -package: clean install - python setup.py clean --all - python setup.py sdist bdist_wheel diff --git a/kedro-docker/kedro_docker/plugin.py b/kedro-docker/kedro_docker/plugin.py index ad63a6de0..d42937952 100644 --- a/kedro-docker/kedro_docker/plugin.py +++ b/kedro-docker/kedro_docker/plugin.py @@ -127,7 +127,7 @@ def docker_init(spark): if KEDRO_VERSION.match(">=0.17.0"): verbose = KedroCliError.VERBOSE_ERROR else: - from kedro.framework.cli.cli import ( # pylint: disable=import-outside-toplevel, no-name-in-module + from kedro.framework.cli.cli import ( # noqa # pylint:disable=import-outside-toplevel, no-name-in-module _VERBOSE as verbose, ) diff --git a/kedro-docker/pyproject.toml b/kedro-docker/pyproject.toml index 28f4e25c8..5d0c96827 100644 --- a/kedro-docker/pyproject.toml +++ b/kedro-docker/pyproject.toml @@ -1 +1,9 @@ [tool.black] + +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 88 +default_section = "THIRDPARTY" diff --git a/kedro-telemetry/.flake8 b/kedro-telemetry/.flake8 deleted file mode 100644 index d3dc8401e..000000000 --- a/kedro-telemetry/.flake8 +++ /dev/null @@ -1,7 +0,0 @@ -# Copied from Black - -[flake8] -ignore = E126, E203, E231, E266, E501, W503 -max-line-length = 80 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 diff --git a/kedro-telemetry/.pre-commit-config.yaml b/kedro-telemetry/.pre-commit-config.yaml index 98175ad45..d0f0e02bc 100644 --- a/kedro-telemetry/.pre-commit-config.yaml +++ b/kedro-telemetry/.pre-commit-config.yaml @@ -16,6 +16,14 @@ repos: - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source. - id: requirements-txt-fixer # Sorts entries in requirements.txt - id: flake8 + files: ^kedro-telemetry/kedro_telemetry/ + args: + - "--max-line-length=88" + - "--max-complexity=18" + - "--max-complexity=18" + - "--select=B,C,E,F,W,T4,B9" + - "--ignore=E126,E203,E231,E266,E501,W503" + - repo: local hooks: @@ -33,7 +41,7 @@ repos: name: "Quick PyLint on tests/*" language: system types: [file, python] - files: ^tests/ + files: ^kedro_telemetry/tests/ entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments stages: [commit] # The same pylint checks, but running on all files. It's for manual run with `make lint` @@ -42,31 +50,32 @@ repos: language: system pass_filenames: false stages: [manual] - entry: pylint --disable=unnecessary-pass kedro_telemetry + entry: pylint --disable=unnecessary-pass kedro-telemetry/kedro_telemetry - id: pylint-tests name: "PyLint on tests/*" language: system pass_filenames: false stages: [manual] - entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments tests + entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-telemetry/tests - id: isort name: "Sort imports" language: system types: [ file, python ] + files: ^kedro-telemetry/ entry: isort - id: black name: "Black" language: system pass_filenames: false - entry: black kedro_telemetry tests + entry: black kedro-telemetry/kedro_telemetry kedro-telemetry/tests - id: secret_scan name: "Secret scan" language: system pass_filenames: false - entry: make secret-scan + entry: make plugin=kedro-telemetry secret-scan - id: bandit name: "Bandit security check" language: system types: [file, python] - exclude: ^tests/ + exclude: ^kedro-telemetry/tests/|^kedro-airflow/|^kedro-docker/ entry: bandit -ll diff --git a/kedro-telemetry/trufflehog-ignore.txt b/kedro-telemetry/trufflehog-ignore.txt deleted file mode 100644 index b43bf86b5..000000000 --- a/kedro-telemetry/trufflehog-ignore.txt +++ /dev/null @@ -1 +0,0 @@ -README.md diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt new file mode 100644 index 000000000..041fc7ffd --- /dev/null +++ b/trufflehog-ignore.txt @@ -0,0 +1 @@ +kedro-telemetry/README.md