[KED-2556] Add CircleCI config

.circleci/config.yml

@@ -0,0 +1,97 @@
+version: 2.1
+
+executors:
+  py38:
+    docker:
+      - image: circleci/python:3.8
+  py37:
+    docker:
+      - image: circleci/python:3.7
+  py36:
+    docker:
+      - image: circleci/python:3.6
+
+commands:
+  setup_requirements:
+    description: Setup requirements
+    parameters:
+      python_version:
+        type: string
+    steps:
+      - checkout
+      - run: |
+          sudo chown -R circleci:circleci /usr/local/bin
+          sudo chown -R circleci:circleci /usr/local/lib/<< parameters.python_version >>/site-packages
+      - run:
+          name: Install pip setuptools
+          command: python -m pip install -U "pip==21.0.1" "setuptools>=38.0" wheel
+      - run:
+          command: |
+            sudo pip install -r test_requirements.txt
+  e2e_run:
+    description: Run `kedro run` end to end tests for all starters
+    steps:
+      - run:
+          command: behave features/run.feature
+  e2e_lint:
+    description: Run `kedro lint` end to end tests for all starters
+    steps:
+      - run:
+          command: behave features/lint.feature
+
+jobs:
+  security_scan:
+    executor: py38
+    steps:
+      - setup_requirements:
+          python_version: "python3.8"
+      - run:
+          command: bandit -ll -r .
+  e2e_38:
+    executor: py38
+    steps:
+      - setup_requirements:
+          python_version: "python3.8"
+      - e2e_run
+  e2e_37:
+    executor: py37
+    steps:
+      - setup_requirements:
+          python_version: "python3.7"
+      - e2e_run
+  e2e_36:
+    executor: py36
+    steps:
+      - setup_requirements:
+          python_version: "python3.6"
+      - e2e_run
+  lint_38:
+    executor: py38
+    steps:
+      - setup_requirements:
+          python_version: "python3.8"
+      - e2e_lint
+  lint_37:
+    executor: py37
+    steps:
+      - setup_requirements:
+          python_version: "python3.7"
+      - e2e_lint
+  lint_36:
+    executor: py36
+    steps:
+      - setup_requirements:
+          python_version: "python3.6"
+      - e2e_lint
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - security_scan
+      - e2e_38
+      - e2e_37
+      - e2e_36
+      - lint_38
+      - lint_37
+      - lint_36

astro-iris/{{ cookiecutter.repo_name }}/docs/source/conf.py

@@ -49,8 +49,7 @@
 
 from kedro.framework.cli.utils import find_stylesheets
 from recommonmark.transform import AutoStructify
-
-from {{ cookiecutter.python_package }} import __version__ as release
+from {{cookiecutter.python_package}} import __version__ as release
 
 # -- Project information -----------------------------------------------------
 

astro-iris/{{ cookiecutter.repo_name }}/src/tests/test_run.py

@@ -43,9 +43,7 @@
 
 @pytest.fixture
 def project_context():
-    return KedroContext(
-        package_name="{{ cookiecutter.python_package }}", project_path=Path.cwd()
-    )
+    return KedroContext(package_name="{{ cookiecutter.python_package }}", project_path=Path.cwd())
 
 
 # The tests below are here for the demonstration purpose

astro-iris/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/pipeline.py

@@ -49,7 +49,7 @@ def create_pipeline(**kwargs):
                     test_x="example_test_x",
                     test_y="example_test_y",
                 ),
-                name="split"
+                name="split",
             )
         ]
     )

astro-iris/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_science/pipeline.py

@@ -44,18 +44,19 @@ def create_pipeline(**kwargs):
                 train_model,
                 ["example_train_x", "example_train_y", "parameters"],
                 "example_model",
-                name="train"
+                name="train",
             ),
             node(
                 predict,
                 dict(model="example_model", test_x="example_test_x"),
                 "example_predictions",
-                name="predict"
+                name="predict",
             ),
             node(
                 report_accuracy,
                 ["example_predictions", "example_test_y"],
                 None,
-                name="report"),
+                name="report",
+            ),
         ]
     )

pandas-iris/features/environment.py → features/environment.py

@@ -51,13 +51,23 @@ def create_new_venv() -> Path:
 def before_scenario(context, scenario):
     """Environment preparation before each test is run.
     """
-
     context.venv_dir = create_new_venv()
     bin_dir = context.venv_dir / "bin"
     context.pip = str(bin_dir / "pip")
     context.python = str(bin_dir / "python")
     context.kedro = str(bin_dir / "kedro")
-    context.starter_path = str(Path(__file__).parents[1])
+    starters_root = Path(__file__).parents[1]
+    starter_names = [
+        "astro-iris",
+        "pandas-iris",
+        "pyspark",
+        "pyspark-iris",
+        "spaceflights",
+    ]
+    starters_paths = {
+        starter: str(starters_root / starter) for starter in starter_names
+    }
+    context.starters_paths = starters_paths
     subprocess.run([context.pip, "install", "-r", "test_requirements.txt"])
     context.temp_dir = Path(tempfile.mkdtemp())
 

features/lint.feature

@@ -0,0 +1,36 @@
+Feature: Lint all starters
+
+  Scenario: Lint astro-iris starter
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter astro-iris
+    And I have installed the Kedro project's dependencies
+    When I lint the project
+    Then I should get a successful exit code
+
+  Scenario: Lint pandas-iris starter
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter pandas-iris
+    And I have installed the Kedro project's dependencies
+    When I lint the project
+    Then I should get a successful exit code
+
+  Scenario: Lint pyspark starter
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter pyspark
+    And I have installed the Kedro project's dependencies
+    When I lint the project
+    Then I should get a successful exit code
+
+  Scenario: Lint pyspark-iris starter
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter pyspark-iris
+    And I have installed the Kedro project's dependencies
+    When I lint the project
+    Then I should get a successful exit code
+
+  Scenario: Lint spaceflights starter
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter spaceflights
+    And I have installed the Kedro project's dependencies
+    When I lint the project
+    Then I should get a successful exit code

features/run.feature

@@ -0,0 +1,36 @@
+Feature: Run all starters
+
+  Scenario: Run a Kedro project created from astro-iris
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter astro-iris
+    And I have installed the Kedro project's dependencies
+    And I have run the Kedro pipeline
+    Then I should get a successful exit code
+
+  Scenario: Run a Kedro project created from pandas-iris
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter pandas-iris
+    And I have installed the Kedro project's dependencies
+    And I have run the Kedro pipeline
+    Then I should get a successful exit code
+
+  Scenario: Run a Kedro project created from pyspark
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter pyspark
+    And I have installed the Kedro project's dependencies
+    And I have executed the CLI command to list Kedro pipelines
+    Then I should get a successful exit code
+
+  Scenario: Run a Kedro project created from pyspark-iris
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter pyspark-iris
+    And I have installed the Kedro project's dependencies
+    And I have executed the CLI command to list Kedro pipelines
+    Then I should get a successful exit code
+
+  Scenario: Run a Kedro project created from spaceflights
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with the starter spaceflights
+    And I have installed the Kedro project's dependencies
+    And I have run the Kedro pipeline
+    Then I should get a successful exit code

pyspark/features/steps/run_steps.py → features/steps/run_steps.py

@@ -1,7 +1,7 @@
 import subprocess
 
 import yaml
-from behave import given, then
+from behave import given, then, when
 
 OK_EXIT_CODE = 0
 
@@ -28,8 +28,8 @@ def create_configuration_file(context):
         yaml.dump(config, config_file, default_flow_style=False)
 
 
-@given("I have run a non-interactive kedro new with the starter")
-def create_project_from_config_file(context):
+@given("I have run a non-interactive kedro new with the starter {starter_name}")
+def create_project_from_config_file(context, starter_name):
     """Behave step to run Kedro new given the config I previously created."""
     res = subprocess.run(
         [
@@ -38,7 +38,7 @@ def create_project_from_config_file(context):
             "--config",
             str(context.config_file),
             "--starter",
-            context.starter_path,
+            context.starters_paths[starter_name],
         ]
     )
     assert res.returncode == OK_EXIT_CODE
@@ -50,14 +50,29 @@ def install_project_dependencies(context):
     assert res.returncode == OK_EXIT_CODE
 
 
-@given("I have executed the CLI command to list Kedro pipelines")
+@given("I have run the Kedro pipeline")
 def run_kedro_pipeline(context):
+    """Behave step to run the newly created Kedro pipeline."""
+    context.result = subprocess.run(
+        [context.kedro, "run"], cwd=context.root_project_dir
+    )
+
+
+@given("I have executed the CLI command to list Kedro pipelines")
+def list_kedro_pipelines(context):
     """Behave step to list Kedro pipelines in a project."""
     context.result = subprocess.run(
         [context.kedro, "pipeline", "list"], cwd=context.root_project_dir
     )
 
 
+@when("I lint the project")
+def lint_project(context):
+    context.result = subprocess.run(
+        [context.kedro, "lint", "--check-only"], cwd=context.root_project_dir
+    )
+
+
 @then("I should get a successful exit code")
 def check_status_code(context):
     if context.result.returncode != OK_EXIT_CODE: