Skip to content

Commit

Permalink
[KED-2556] Add CircleCI config
Browse files Browse the repository at this point in the history
  • Loading branch information
Lorena Bălan authored May 10, 2021
2 parents 9cdfe96 + c1fe637 commit 81b5e07
Show file tree
Hide file tree
Showing 50 changed files with 242 additions and 691 deletions.
97 changes: 97 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
version: 2.1

executors:
py38:
docker:
- image: circleci/python:3.8
py37:
docker:
- image: circleci/python:3.7
py36:
docker:
- image: circleci/python:3.6

commands:
setup_requirements:
description: Setup requirements
parameters:
python_version:
type: string
steps:
- checkout
- run: |
sudo chown -R circleci:circleci /usr/local/bin
sudo chown -R circleci:circleci /usr/local/lib/<< parameters.python_version >>/site-packages
- run:
name: Install pip setuptools
command: python -m pip install -U "pip==21.0.1" "setuptools>=38.0" wheel
- run:
command: |
sudo pip install -r test_requirements.txt
e2e_run:
description: Run `kedro run` end to end tests for all starters
steps:
- run:
command: behave features/run.feature
e2e_lint:
description: Run `kedro lint` end to end tests for all starters
steps:
- run:
command: behave features/lint.feature

jobs:
security_scan:
executor: py38
steps:
- setup_requirements:
python_version: "python3.8"
- run:
command: bandit -ll -r .
e2e_38:
executor: py38
steps:
- setup_requirements:
python_version: "python3.8"
- e2e_run
e2e_37:
executor: py37
steps:
- setup_requirements:
python_version: "python3.7"
- e2e_run
e2e_36:
executor: py36
steps:
- setup_requirements:
python_version: "python3.6"
- e2e_run
lint_38:
executor: py38
steps:
- setup_requirements:
python_version: "python3.8"
- e2e_lint
lint_37:
executor: py37
steps:
- setup_requirements:
python_version: "python3.7"
- e2e_lint
lint_36:
executor: py36
steps:
- setup_requirements:
python_version: "python3.6"
- e2e_lint

workflows:
version: 2
build:
jobs:
- security_scan
- e2e_38
- e2e_37
- e2e_36
- lint_38
- lint_37
- lint_36
3 changes: 1 addition & 2 deletions astro-iris/{{ cookiecutter.repo_name }}/docs/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@

from kedro.framework.cli.utils import find_stylesheets
from recommonmark.transform import AutoStructify

from {{ cookiecutter.python_package }} import __version__ as release
from {{cookiecutter.python_package}} import __version__ as release

# -- Project information -----------------------------------------------------

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,7 @@

@pytest.fixture
def project_context():
return KedroContext(
package_name="{{ cookiecutter.python_package }}", project_path=Path.cwd()
)
return KedroContext(package_name="{{ cookiecutter.python_package }}", project_path=Path.cwd())


# The tests below are here for the demonstration purpose
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def create_pipeline(**kwargs):
test_x="example_test_x",
test_y="example_test_y",
),
name="split"
name="split",
)
]
)
Original file line number Diff line number Diff line change
Expand Up @@ -44,18 +44,19 @@ def create_pipeline(**kwargs):
train_model,
["example_train_x", "example_train_y", "parameters"],
"example_model",
name="train"
name="train",
),
node(
predict,
dict(model="example_model", test_x="example_test_x"),
"example_predictions",
name="predict"
name="predict",
),
node(
report_accuracy,
["example_predictions", "example_test_y"],
None,
name="report"),
name="report",
),
]
)
14 changes: 12 additions & 2 deletions pandas-iris/features/environment.py → features/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,23 @@ def create_new_venv() -> Path:
def before_scenario(context, scenario):
"""Environment preparation before each test is run.
"""

context.venv_dir = create_new_venv()
bin_dir = context.venv_dir / "bin"
context.pip = str(bin_dir / "pip")
context.python = str(bin_dir / "python")
context.kedro = str(bin_dir / "kedro")
context.starter_path = str(Path(__file__).parents[1])
starters_root = Path(__file__).parents[1]
starter_names = [
"astro-iris",
"pandas-iris",
"pyspark",
"pyspark-iris",
"spaceflights",
]
starters_paths = {
starter: str(starters_root / starter) for starter in starter_names
}
context.starters_paths = starters_paths
subprocess.run([context.pip, "install", "-r", "test_requirements.txt"])
context.temp_dir = Path(tempfile.mkdtemp())

Expand Down
36 changes: 36 additions & 0 deletions features/lint.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
Feature: Lint all starters

Scenario: Lint astro-iris starter
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter astro-iris
And I have installed the Kedro project's dependencies
When I lint the project
Then I should get a successful exit code

Scenario: Lint pandas-iris starter
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter pandas-iris
And I have installed the Kedro project's dependencies
When I lint the project
Then I should get a successful exit code

Scenario: Lint pyspark starter
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter pyspark
And I have installed the Kedro project's dependencies
When I lint the project
Then I should get a successful exit code

Scenario: Lint pyspark-iris starter
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter pyspark-iris
And I have installed the Kedro project's dependencies
When I lint the project
Then I should get a successful exit code

Scenario: Lint spaceflights starter
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter spaceflights
And I have installed the Kedro project's dependencies
When I lint the project
Then I should get a successful exit code
36 changes: 36 additions & 0 deletions features/run.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
Feature: Run all starters

Scenario: Run a Kedro project created from astro-iris
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter astro-iris
And I have installed the Kedro project's dependencies
And I have run the Kedro pipeline
Then I should get a successful exit code

Scenario: Run a Kedro project created from pandas-iris
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter pandas-iris
And I have installed the Kedro project's dependencies
And I have run the Kedro pipeline
Then I should get a successful exit code

Scenario: Run a Kedro project created from pyspark
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter pyspark
And I have installed the Kedro project's dependencies
And I have executed the CLI command to list Kedro pipelines
Then I should get a successful exit code

Scenario: Run a Kedro project created from pyspark-iris
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter pyspark-iris
And I have installed the Kedro project's dependencies
And I have executed the CLI command to list Kedro pipelines
Then I should get a successful exit code

Scenario: Run a Kedro project created from spaceflights
Given I have prepared a config file
And I have run a non-interactive kedro new with the starter spaceflights
And I have installed the Kedro project's dependencies
And I have run the Kedro pipeline
Then I should get a successful exit code
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import subprocess

import yaml
from behave import given, then
from behave import given, then, when

OK_EXIT_CODE = 0

Expand All @@ -28,8 +28,8 @@ def create_configuration_file(context):
yaml.dump(config, config_file, default_flow_style=False)


@given("I have run a non-interactive kedro new with the starter")
def create_project_from_config_file(context):
@given("I have run a non-interactive kedro new with the starter {starter_name}")
def create_project_from_config_file(context, starter_name):
"""Behave step to run Kedro new given the config I previously created."""
res = subprocess.run(
[
Expand All @@ -38,7 +38,7 @@ def create_project_from_config_file(context):
"--config",
str(context.config_file),
"--starter",
context.starter_path,
context.starters_paths[starter_name],
]
)
assert res.returncode == OK_EXIT_CODE
Expand All @@ -50,14 +50,29 @@ def install_project_dependencies(context):
assert res.returncode == OK_EXIT_CODE


@given("I have executed the CLI command to list Kedro pipelines")
@given("I have run the Kedro pipeline")
def run_kedro_pipeline(context):
"""Behave step to run the newly created Kedro pipeline."""
context.result = subprocess.run(
[context.kedro, "run"], cwd=context.root_project_dir
)


@given("I have executed the CLI command to list Kedro pipelines")
def list_kedro_pipelines(context):
"""Behave step to list Kedro pipelines in a project."""
context.result = subprocess.run(
[context.kedro, "pipeline", "list"], cwd=context.root_project_dir
)


@when("I lint the project")
def lint_project(context):
context.result = subprocess.run(
[context.kedro, "lint", "--check-only"], cwd=context.root_project_dir
)


@then("I should get a successful exit code")
def check_status_code(context):
if context.result.returncode != OK_EXIT_CODE:
Expand Down
8 changes: 0 additions & 8 deletions pandas-iris/features/run.feature

This file was deleted.

68 changes: 0 additions & 68 deletions pandas-iris/features/steps/run_steps.py

This file was deleted.

5 changes: 0 additions & 5 deletions pandas-iris/test_requirements.txt

This file was deleted.

Loading

0 comments on commit 81b5e07

Please sign in to comment.