From 8cc98cb9e4c249fca26a10036255173145e46dea Mon Sep 17 00:00:00 2001 From: Kumar Gaurav Sharma Date: Sat, 10 Jul 2021 01:20:15 -0700 Subject: [PATCH 1/4] Declarative integration test framework --- test/declarative_test_fwk/helper.py | 180 ++++++++++++++++++++++++++++ test/declarative_test_fwk/loader.py | 67 +++++++++++ test/declarative_test_fwk/model.py | 80 +++++++++++++ test/declarative_test_fwk/runner.py | 135 +++++++++++++++++++++ test/e2e/__init__.py | 3 + test/e2e/bootstrap_resources.py | 12 ++ test/e2e/conftest.py | 6 +- test/e2e/declarative_tests.py | 75 ++++++++++++ test/e2e/scenarios/scenario1.yaml | 47 ++++++++ 9 files changed, 603 insertions(+), 2 deletions(-) create mode 100644 test/declarative_test_fwk/helper.py create mode 100644 test/declarative_test_fwk/loader.py create mode 100644 test/declarative_test_fwk/model.py create mode 100644 test/declarative_test_fwk/runner.py create mode 100644 test/e2e/declarative_tests.py create mode 100644 test/e2e/scenarios/scenario1.yaml diff --git a/test/declarative_test_fwk/helper.py b/test/declarative_test_fwk/helper.py new file mode 100644 index 00000000..dd9dda9b --- /dev/null +++ b/test/declarative_test_fwk/helper.py @@ -0,0 +1,180 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""Helper for Declarative tests framework for custom resources +""" + +import logging +from time import sleep +from acktest.k8s import resource as k8s + +input_replacements_dict = dict() +TEST_HELPERS = dict() + + +def input_replacements(provider): + """ + Decorator to discover input replacements + :param provider: function that returns input replacements + :return: provider + """ + global input_replacements_dict + input_replacements_dict = provider() + return provider + + +def resource_helper(resource_kind: str): + """ + Decorator to discover Custom Resource Helper + :param resource_kind: custom resource kind + """ + def registrar(cls): + TEST_HELPERS[resource_kind.lower()] = cls + logging.info(f"Registered ResourceHepler: {cls.__name__} for custom resource kind: {resource_kind}") + return registrar + + +class ResourceHelper: + """ + Provides generic verb (create, patch, delete) methods for custom resources. + Keep its methods stateless. Methods are on instance to allow specialization. + """ + DEFAULT_WAIT_SECS = 30 + + def create(self, input_data: dict): + """ + Creates custom resource + :param input_data: resource details + :return: k8s.CustomResourceReference, created custom resource + """ + reference = self.custom_resource_reference(input_data) + _ = k8s.create_custom_resource(reference, input_data) + resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) + assert resource is not None + return (reference, resource) + + def patch(self, input_data: dict): + """ + Patches custom resource + :param input_data: resource patch details + :return: k8s.CustomResourceReference, created custom resource + """ + reference = self.custom_resource_reference(input_data) + _ = k8s.patch_custom_resource(reference, input_data) + sleep(self.DEFAULT_WAIT_SECS) # required as controller has likely not placed the resource in modifying + resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) + assert resource is not None + return (reference, resource) + + def delete(self, reference: k8s.CustomResourceReference): + """ + Deletes custom resource and waits for delete completion + :param reference: resource reference + :return: None + """ + resource = k8s.get_resource(reference) + if not resource: + logging.warning(f"ResourceReference {reference} not found. Not invoking k8s delete api.") + return + + k8s.delete_custom_resource(reference) + sleep(self.DEFAULT_WAIT_SECS) + self.wait_for_delete(reference) # throws exception if wait fails + + def assert_expectations(self, expectations: dict, reference: k8s.CustomResourceReference): + """ + Asserts custom resource reference against supplied expectations + :param expectations: expectations to assert + :param reference: custom resource reference + :return: None + """ + # condition assertion contains wait logic + self._assert_conditions(expectations, reference) + + # conditions expectations met, now check current resource against expectations + resource = k8s.get_resource(reference) + self.assert_items(expectations.get("status"), resource.get("status")) + + # self._assert_state(expectations.get("spec"), resource) # uncomment to support spec assertions + + def _assert_conditions(self, expectations: dict, reference: k8s.CustomResourceReference): + expect_conditions: dict = {} + if "status" in expectations and "conditions" in expectations["status"]: + expect_conditions = expectations["status"]["conditions"] + + for (condition_name, condition_value) in expect_conditions.items(): + assert k8s.wait_on_condition(reference, condition_name, condition_value, wait_periods=30) + + def assert_items(self, expectations: dict, state: dict): + """ + Asserts state against supplied expectations + Override it as needed for custom verifications + :param expectations: dictionary with items to assert in state + :param state: dictionary with items + :return: None + """ + if not expectations: + return + if not state: + assert expectations == state + + for (property, value) in expectations.items(): + # conditions are processed separately + if property == "conditions": + continue + assert (property, value) == (property, state.get(property)) + + def custom_resource_reference(self, input_data: dict) -> k8s.CustomResourceReference: + """ + Helper method to provide k8s.CustomResourceReference for supplied input + :param input_data: custom resource input data + :return: k8s.CustomResourceReference + """ + resource_plural = self.resource_plural(input_data.get("kind")) + resource_name = input_data.get("metadata").get("name") + crd_group = input_replacements_dict.get("CRD_GROUP") + crd_version = input_replacements_dict.get("CRD_VERSION") + + reference = k8s.CustomResourceReference( + crd_group, crd_version, resource_plural, resource_name, namespace="default") + return reference + + def wait_for_delete(self, reference: k8s.CustomResourceReference): + """ + Override this method to implement custom resource delete logic. + :param reference: custom resource reference + :return: None + """ + logging.debug(f"No-op wait_for_delete()") + + def resource_plural(self, resource_kind: str) -> str: + """ + Provide plural string for supplied custom resource kind + Override as needed + :param resource_kind: custom resource kind + :return: plural string + """ + return resource_kind.lower() + "s" + + +def get_resource_helper(resource_kind: str) -> ResourceHelper: + """ + Provides ResourceHelper for supplied custom resource kind + If no helper is registered for the supplied resource kind then returns default ResourceHelper + :param resource_kind: custom resource kind + :return: custom resource helper + """ + helper_cls = TEST_HELPERS.get(resource_kind.lower()) + if helper_cls: + return helper_cls() + return ResourceHelper() diff --git a/test/declarative_test_fwk/loader.py b/test/declarative_test_fwk/loader.py new file mode 100644 index 00000000..627bf705 --- /dev/null +++ b/test/declarative_test_fwk/loader.py @@ -0,0 +1,67 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""Test Scenarios loader for Declarative tests framework for custom resources +""" + +from declarative_test_fwk import model, helper +import pytest +import os +from typing import Iterable +from pathlib import Path +from acktest.resources import load_resource_file, random_suffix_name + + +def scenarios(scenarios_directory: Path) -> Iterable: + """ + Loads scenarios from given directory + :param scenarios_directory: directory containing scenarios yaml files + :return: Iterable scenarios + """ + for scenario_file in os.listdir(scenarios_directory): + if not scenario_file.endswith(".yaml"): + continue + scenario_name = scenario_file.split(".yaml")[0] + replacements = helper.input_replacements_dict.copy() + replacements["RANDOM_SUFFIX"] = random_suffix_name("", 32) + scenario = model.Scenario(load_resource_file( + scenarios_directory, scenario_name, additional_replacements=replacements)) + yield pytest.param(scenario, marks=marks(scenario)) + + +def idfn(scenario: model.Scenario) -> str: + """ + Provides scenario test id + :param scenario: test scenario + :return: scenario test id string + """ + return scenario.id() + + +def marks(scenario: model.Scenario) -> list: + """ + Provides pytest markers for the scenario + :param scenario: test scenario + :return: markers for the scenario + """ + markers = [] + for usecase in scenario.usecases(): + markers.append(pytest.mark.usecase(arg=usecase)) + for mark in scenario.marks(): + if mark == "canary": + markers.append(pytest.mark.canary) + elif mark == "slow": + markers.append(pytest.mark.slow) + elif mark == "blocked": + markers.append(pytest.mark.blocked) + return markers diff --git a/test/declarative_test_fwk/model.py b/test/declarative_test_fwk/model.py new file mode 100644 index 00000000..1a6f9017 --- /dev/null +++ b/test/declarative_test_fwk/model.py @@ -0,0 +1,80 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""Model for Declarative tests framework for custom resources +""" + +from declarative_test_fwk import helper + + +class Step: + """ + Represents a declarative test step + """ + indent = "\t\t" + + def __init__(self, config: dict): + self.config = config + + self.verb = None + self.input_data = None + self.expectations = None + self.resource_helper = None + + # (k8s.CustomResourceReference, ko) to teardown + self.teardown_list = [] + + supported_verbs=["create", "patch", "delete"] + for verb in supported_verbs: + if verb not in self.config: + continue + self.verb = verb + self.input_data = self.config.get(verb) + break + + if self.input_data: + self.expectations = self.config.get("expect") + self.resource_helper = helper.get_resource_helper(self.input_data.get("kind")) + + def id(self) -> str: + return self.config.get("id", "") + + def description(self) -> str: + return self.config.get("description", "") + + +class Scenario: + """ + Represents a declarative test scenario with steps + """ + + def __init__(self, config: dict): + self.config = config + self.test_steps = [] + for step in self.config.get("steps", []): + self.test_steps.append(Step(step)) + + def id(self) -> str: + return self.config.get("id", "") + + def description(self) -> str: + return self.config.get("description", "") + + def usecases(self) -> list: + return self.config.get("usecases", []) + + def marks(self) -> list: + return self.config.get("marks", []) + + def steps(self): + return self.test_steps diff --git a/test/declarative_test_fwk/runner.py b/test/declarative_test_fwk/runner.py new file mode 100644 index 00000000..0f87ab59 --- /dev/null +++ b/test/declarative_test_fwk/runner.py @@ -0,0 +1,135 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""Runner for Declarative tests framework scenarios for custom resources +""" + +from declarative_test_fwk import model +import pytest +import sys +import logging +from acktest.k8s import resource as k8s + + +def run(scenario: model.Scenario) -> None: + """ + Runs steps in the scenario + """ + if not scenario: + return + + logging.info(f"Execute: Scenario: {scenario.id()}") + for step in scenario.steps(): + run_step(step) + + +def teardown(scenario: model.Scenario) -> None: + """ + Teardown steps in the scenario in reverse order + """ + if not scenario: + return + + logging.info(f"Teardown: Scenario: {scenario.id()}") + teardown_failures = [] + # tear down steps in reverse order + for step in reversed(scenario.steps()): + try: + teardown_step(step) + except: + error = f"Failed to teardown Step: {step.id()}. " \ + f"Unexpected error: {sys.exc_info()[0]}" + teardown_failures.append(error) + logging.debug(error) + + if len(teardown_failures) != 0: + teardown_failures.insert(0, f"Failures during teardown. Scenario: {scenario.id()}") + failures = "\n\t- ".join(teardown_failures) + logging.error(failures) + pytest.fail(failures) + + +def run_step(step: model.Step) -> None: + """ + Runs step + """ + if not step: + return + + if not step.verb: + logging.warning( + f"skipped: Step: {step.id()}. No matching verb found." + f" Supported verbs: create, patch, delete.") + return + + if step.verb == "create": + create_resource(step) + elif step.verb == "patch": + patch_resource(step) + elif step.verb == "delete": + pass + assert_expectations(step) + + +def create_resource(step: model.Step) -> None: + logging.debug(f"create: Step: {step.id()}") + if not step.input_data: + return + + (reference, ko) = step.resource_helper.create(step.input_data) + # track created reference to teardown later + step.teardown_list.append((reference, ko)) + + +def patch_resource(step: model.Step) -> None: + logging.debug(f"patch: Step: {step.id()}") + if not step.input_data: + return + + (reference, ko) = step.resource_helper.patch(step.input_data) + # no need to teardown patched reference, its creator should tear it down. + + +def delete_resource(step: model.Step, reference: k8s.CustomResourceReference = None) -> None: + if not reference: + logging.debug(f"delete: Step: {step.id()}") + reference = step.resource_helper.custom_resource_reference(step.input_data) + + step.resource_helper.delete(reference) + + +def assert_expectations(step: model.Step) -> None: + logging.debug(f"assert: Step: {step.id()}") + if not step.expectations: + return + + resource_helper = step.resource_helper + reference = resource_helper.custom_resource_reference(step.input_data) + resource_helper.assert_expectations(step.expectations, reference) + + +def teardown_step(step: model.Step) -> None: + """ + Teardown resources from the step + """ + if not step or len(step.teardown_list) == 0: + return + + logging.info(f"teardown: Step: {step.id()}") + + for (reference, _) in step.teardown_list: + if reference: + delete_resource(reference) + + # clear list + step.teardown_list = [] diff --git a/test/e2e/__init__.py b/test/e2e/__init__.py index eb244167..ffbb4efe 100644 --- a/test/e2e/__init__.py +++ b/test/e2e/__init__.py @@ -26,6 +26,9 @@ bootstrap_directory = Path(__file__).parent resource_directory = Path(__file__).parent / "resources" +scenarios_directory = Path(__file__).parent / "scenarios" + + def load_elasticache_resource(resource_name: str, additional_replacements: Dict[str, Any] = {}): """ Overrides the default `load_resource_file` to access the specific resources directory for the current service. diff --git a/test/e2e/bootstrap_resources.py b/test/e2e/bootstrap_resources.py index e5e2ef4e..47117f8b 100644 --- a/test/e2e/bootstrap_resources.py +++ b/test/e2e/bootstrap_resources.py @@ -34,8 +34,20 @@ class TestBootstrapResources: CWLogGroup2: str CPGName: str + def replacement_dict(self): + return { + "SNS_TOPIC_ARN": self.SnsTopicARN, + "SG_ID": self.SecurityGroupID, + "USERGROUP_ID": self.UserGroupID, + "KMS_KEY_ID": self.KmsKeyID, + "SNAPSHOT_NAME": self.SnapshotName, + "NON_DEFAULT_USER": self.SnapshotName + } + + _bootstrap_resources = None + def get_bootstrap_resources(bootstrap_file_name: str = "bootstrap.yaml"): global _bootstrap_resources if _bootstrap_resources is None: diff --git a/test/e2e/conftest.py b/test/e2e/conftest.py index b05fd51e..25107c74 100644 --- a/test/e2e/conftest.py +++ b/test/e2e/conftest.py @@ -11,9 +11,7 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -import os import pytest - from acktest import k8s @@ -29,6 +27,9 @@ def pytest_configure(config): config.addinivalue_line( "markers", "service(arg): mark test associated with a given service" ) + config.addinivalue_line( + "markers", "usecase(arg): mark test associated with a given usecase" + ) config.addinivalue_line( "markers", "slow: mark test as slow to run" ) @@ -48,6 +49,7 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip_slow) if "blocked" in item.keywords and not config.getoption("--runblocked"): item.add_marker(skip_blocked) + # TODO: choose test per 'usecase' selector # Provide a k8s client to interact with the integration test cluster diff --git a/test/e2e/declarative_tests.py b/test/e2e/declarative_tests.py new file mode 100644 index 00000000..a94bc627 --- /dev/null +++ b/test/e2e/declarative_tests.py @@ -0,0 +1,75 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +""" +Tests for custom resources. +Uses declarative tests framework for custom resources. + +To add test: add scenario yaml to scenarios/ directory. +""" + +from declarative_test_fwk import helper, loader, runner + +import pytest +import boto3 +import logging + +from e2e import service_marker, scenarios_directory, CRD_VERSION, CRD_GROUP, SERVICE_NAME +from e2e.bootstrap_resources import get_bootstrap_resources +from acktest.k8s import resource as k8s + + +@helper.input_replacements +def input_replacements(): + """ + Input replacements for test scenarios + """ + replacements = get_bootstrap_resources().replacement_dict() + replacements["CRD_VERSION"] = CRD_VERSION + replacements["CRD_GROUP"] = CRD_GROUP + replacements["SERVICE_NAME"] = SERVICE_NAME + return replacements + + +@helper.resource_helper("ReplicationGroup") +class ReplicationGroupHelper(helper.ResourceHelper): + """ + Helper for replication group scenarios + """ + def wait_for_delete(self, reference: k8s.CustomResourceReference): + logging.debug(f"ReplicationGroupHelper - wait_for_delete()") + ec = boto3.client("elasticache") + waiter = ec.get_waiter('replication_group_deleted') + # throws exception if wait fails + waiter.wait(ReplicationGroupId=reference.name) + + +@pytest.fixture(params=loader.scenarios(scenarios_directory), ids=loader.idfn) +def scenario(request): + """ + Parameterized fixture + Provides scenarios to execute + Supports parallel execution of scenarios + """ + s = request.param + yield s + runner.teardown(s) + + +@service_marker +class TestSuite: + """ + Declarative scenarios based test suite + """ + def test_scenario(self, scenario): + runner.run(scenario) diff --git a/test/e2e/scenarios/scenario1.yaml b/test/e2e/scenarios/scenario1.yaml new file mode 100644 index 00000000..ce9cff36 --- /dev/null +++ b/test/e2e/scenarios/scenario1.yaml @@ -0,0 +1,47 @@ +id: "RG_CMD_basic_create_update" +description: "In this test we create RG with absolutely minimum configuration and try to update it." +usecases: + - replication_group + - cmd +#marks: +# - slow +# - blocked +steps: + - id: "create_CMD_replication_group" + description: "Initial config" + create: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: ReplicationGroup + metadata: + name: test$RANDOM_SUFFIX + spec: + engine: redis + replicationGroupID: test$RANDOM_SUFFIX + replicationGroupDescription: Basic create and update of CMD RG + cacheNodeType: cache.t3.micro + numNodeGroups: 1 + expect: + status: + conditions: + ACK.ResourceSynced: "True" + status: "available" + - id: "invalid_cacheNodeType_on_CMD_replication_group_causes_Terminal_condition" + description: "Negative test include transitEncryptionEnabled" + patch: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: ReplicationGroup + metadata: + name: test$RANDOM_SUFFIX + spec: + cacheNodeType: cache.micro # invalid value + expect: + status: + conditions: + ACK.Terminal: "True" + - id: "delete_CMD_RG" + description: "Delete cluster mode disabled replication group" + delete: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: ReplicationGroup + metadata: + name: test$RANDOM_SUFFIX From 15521d9c1a32118e50a7176f089b3eb932a7397f Mon Sep 17 00:00:00 2001 From: Kumar Gaurav Sharma Date: Sun, 11 Jul 2021 16:28:10 -0700 Subject: [PATCH 2/4] Use fixture for service_bootstrap resources and input replacements --- test/declarative_test_fwk/helper.py | 29 ++++++--------- test/declarative_test_fwk/loader.py | 40 ++++++++++++++------- test/declarative_test_fwk/model.py | 8 +++-- test/declarative_test_fwk/runner.py | 8 ++--- test/e2e/bootstrap_resources.py | 11 ------ test/e2e/declarative_tests.py | 56 ++++++++++++++++++----------- 6 files changed, 82 insertions(+), 70 deletions(-) diff --git a/test/declarative_test_fwk/helper.py b/test/declarative_test_fwk/helper.py index dd9dda9b..7a2ab3a1 100644 --- a/test/declarative_test_fwk/helper.py +++ b/test/declarative_test_fwk/helper.py @@ -18,21 +18,9 @@ from time import sleep from acktest.k8s import resource as k8s -input_replacements_dict = dict() TEST_HELPERS = dict() -def input_replacements(provider): - """ - Decorator to discover input replacements - :param provider: function that returns input replacements - :return: provider - """ - global input_replacements_dict - input_replacements_dict = provider() - return provider - - def resource_helper(resource_kind: str): """ Decorator to discover Custom Resource Helper @@ -51,25 +39,27 @@ class ResourceHelper: """ DEFAULT_WAIT_SECS = 30 - def create(self, input_data: dict): + def create(self, input_data: dict, input_replacements: dict = {}): """ Creates custom resource :param input_data: resource details + :param input_replacements: input replacements :return: k8s.CustomResourceReference, created custom resource """ - reference = self.custom_resource_reference(input_data) + reference = self.custom_resource_reference(input_data, input_replacements) _ = k8s.create_custom_resource(reference, input_data) resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) assert resource is not None return (reference, resource) - def patch(self, input_data: dict): + def patch(self, input_data: dict, input_replacements: dict = {}): """ Patches custom resource :param input_data: resource patch details + :param input_replacements: input replacements :return: k8s.CustomResourceReference, created custom resource """ - reference = self.custom_resource_reference(input_data) + reference = self.custom_resource_reference(input_data, input_replacements) _ = k8s.patch_custom_resource(reference, input_data) sleep(self.DEFAULT_WAIT_SECS) # required as controller has likely not placed the resource in modifying resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) @@ -134,16 +124,17 @@ def assert_items(self, expectations: dict, state: dict): continue assert (property, value) == (property, state.get(property)) - def custom_resource_reference(self, input_data: dict) -> k8s.CustomResourceReference: + def custom_resource_reference(self, input_data: dict, input_replacements: dict = {}) -> k8s.CustomResourceReference: """ Helper method to provide k8s.CustomResourceReference for supplied input :param input_data: custom resource input data + :param input_replacements: input replacements :return: k8s.CustomResourceReference """ resource_plural = self.resource_plural(input_data.get("kind")) resource_name = input_data.get("metadata").get("name") - crd_group = input_replacements_dict.get("CRD_GROUP") - crd_version = input_replacements_dict.get("CRD_VERSION") + crd_group = input_replacements.get("CRD_GROUP") + crd_version = input_replacements.get("CRD_VERSION") reference = k8s.CustomResourceReference( crd_group, crd_version, resource_plural, resource_name, namespace="default") diff --git a/test/declarative_test_fwk/loader.py b/test/declarative_test_fwk/loader.py index 627bf705..0c4dfe16 100644 --- a/test/declarative_test_fwk/loader.py +++ b/test/declarative_test_fwk/loader.py @@ -19,33 +19,47 @@ import os from typing import Iterable from pathlib import Path +from os.path import isfile, join from acktest.resources import load_resource_file, random_suffix_name -def scenarios(scenarios_directory: Path) -> Iterable: +def list_scenarios(scenarios_directory: Path) -> Iterable[Path]: """ - Loads scenarios from given directory + Lists test scenarios from given directory :param scenarios_directory: directory containing scenarios yaml files :return: Iterable scenarios """ + scenarios_list = [] for scenario_file in os.listdir(scenarios_directory): - if not scenario_file.endswith(".yaml"): + scenario_file_full_path = join(scenarios_directory, scenario_file) + if not isfile(scenario_file_full_path) or not scenario_file.endswith(".yaml"): continue - scenario_name = scenario_file.split(".yaml")[0] - replacements = helper.input_replacements_dict.copy() - replacements["RANDOM_SUFFIX"] = random_suffix_name("", 32) - scenario = model.Scenario(load_resource_file( - scenarios_directory, scenario_name, additional_replacements=replacements)) - yield pytest.param(scenario, marks=marks(scenario)) + scenarios_list.append(Path(scenario_file_full_path)) + return scenarios_list -def idfn(scenario: model.Scenario) -> str: +def load_scenario(scenario_file: Path, replacements: dict = {}) -> Iterable: """ - Provides scenario test id - :param scenario: test scenario + Loads scenario from given scenario_file + :param scenario_file: yaml file containing scenarios + :param replacements: input replacements + :return: Iterable scenarios + """ + scenario_name = scenario_file.name.split(".yaml")[0] + replacements = replacements.copy() + replacements["RANDOM_SUFFIX"] = random_suffix_name("", 32) + scenario = model.Scenario(load_resource_file( + scenario_file.parent, scenario_name, additional_replacements=replacements), replacements) + yield pytest.param(scenario, marks=marks(scenario)) + + +def idfn(scenario_file_full_path: Path) -> str: + """ + Provides scenario file name as scenario test id + :param scenario: test scenario file path :return: scenario test id string """ - return scenario.id() + return scenario_file_full_path.name def marks(scenario: model.Scenario) -> list: diff --git a/test/declarative_test_fwk/model.py b/test/declarative_test_fwk/model.py index 1a6f9017..38fe7c2f 100644 --- a/test/declarative_test_fwk/model.py +++ b/test/declarative_test_fwk/model.py @@ -23,8 +23,9 @@ class Step: """ indent = "\t\t" - def __init__(self, config: dict): + def __init__(self, config: dict, replacements: dict = {}): self.config = config + self.replacements = replacements self.verb = None self.input_data = None @@ -58,11 +59,12 @@ class Scenario: Represents a declarative test scenario with steps """ - def __init__(self, config: dict): + def __init__(self, config: dict, replacements: dict = {}): self.config = config self.test_steps = [] + self.replacements = replacements for step in self.config.get("steps", []): - self.test_steps.append(Step(step)) + self.test_steps.append(Step(step, replacements)) def id(self) -> str: return self.config.get("id", "") diff --git a/test/declarative_test_fwk/runner.py b/test/declarative_test_fwk/runner.py index 0f87ab59..6956e9bc 100644 --- a/test/declarative_test_fwk/runner.py +++ b/test/declarative_test_fwk/runner.py @@ -86,7 +86,7 @@ def create_resource(step: model.Step) -> None: if not step.input_data: return - (reference, ko) = step.resource_helper.create(step.input_data) + (reference, ko) = step.resource_helper.create(step.input_data, step.replacements) # track created reference to teardown later step.teardown_list.append((reference, ko)) @@ -96,14 +96,14 @@ def patch_resource(step: model.Step) -> None: if not step.input_data: return - (reference, ko) = step.resource_helper.patch(step.input_data) + (reference, ko) = step.resource_helper.patch(step.input_data, step.replacements) # no need to teardown patched reference, its creator should tear it down. def delete_resource(step: model.Step, reference: k8s.CustomResourceReference = None) -> None: if not reference: logging.debug(f"delete: Step: {step.id()}") - reference = step.resource_helper.custom_resource_reference(step.input_data) + reference = step.resource_helper.custom_resource_reference(step.input_data, step.replacements) step.resource_helper.delete(reference) @@ -114,7 +114,7 @@ def assert_expectations(step: model.Step) -> None: return resource_helper = step.resource_helper - reference = resource_helper.custom_resource_reference(step.input_data) + reference = resource_helper.custom_resource_reference(step.input_data, step.replacements) resource_helper.assert_expectations(step.expectations, reference) diff --git a/test/e2e/bootstrap_resources.py b/test/e2e/bootstrap_resources.py index 47117f8b..9572fa75 100644 --- a/test/e2e/bootstrap_resources.py +++ b/test/e2e/bootstrap_resources.py @@ -34,17 +34,6 @@ class TestBootstrapResources: CWLogGroup2: str CPGName: str - def replacement_dict(self): - return { - "SNS_TOPIC_ARN": self.SnsTopicARN, - "SG_ID": self.SecurityGroupID, - "USERGROUP_ID": self.UserGroupID, - "KMS_KEY_ID": self.KmsKeyID, - "SNAPSHOT_NAME": self.SnapshotName, - "NON_DEFAULT_USER": self.SnapshotName - } - - _bootstrap_resources = None diff --git a/test/e2e/declarative_tests.py b/test/e2e/declarative_tests.py index a94bc627..c27d1beb 100644 --- a/test/e2e/declarative_tests.py +++ b/test/e2e/declarative_tests.py @@ -24,23 +24,9 @@ import boto3 import logging -from e2e import service_marker, scenarios_directory, CRD_VERSION, CRD_GROUP, SERVICE_NAME -from e2e.bootstrap_resources import get_bootstrap_resources +from e2e import service_bootstrap, service_cleanup, service_marker, scenarios_directory, CRD_VERSION, CRD_GROUP, SERVICE_NAME from acktest.k8s import resource as k8s - -@helper.input_replacements -def input_replacements(): - """ - Input replacements for test scenarios - """ - replacements = get_bootstrap_resources().replacement_dict() - replacements["CRD_VERSION"] = CRD_VERSION - replacements["CRD_GROUP"] = CRD_GROUP - replacements["SERVICE_NAME"] = SERVICE_NAME - return replacements - - @helper.resource_helper("ReplicationGroup") class ReplicationGroupHelper(helper.ResourceHelper): """ @@ -54,16 +40,46 @@ def wait_for_delete(self, reference: k8s.CustomResourceReference): waiter.wait(ReplicationGroupId=reference.name) -@pytest.fixture(params=loader.scenarios(scenarios_directory), ids=loader.idfn) -def scenario(request): +@pytest.fixture(scope="session") +def input_replacements(): + """ + Session scoped fixture to bootstrap service resources and teardown + provides input replacements for test scenarios. + Eliminates the need for: + - bootstrap.yaml and + - call to /service_bootstrap.py from test-infra + - call to /service_cleanup.py from test-infra + """ + + resources_dict = service_bootstrap.service_bootstrap() + replacements = { + "CRD_VERSION": CRD_VERSION, + "CRD_GROUP": CRD_GROUP, + "SERVICE_NAME": SERVICE_NAME, + "SNS_TOPIC_ARN": resources_dict.get("SnsTopicARN"), + "SG_ID": resources_dict.get("SecurityGroupID"), + "USERGROUP_ID": resources_dict.get("UserGroupID"), + "KMS_KEY_ID": resources_dict.get("KmsKeyID"), + "SNAPSHOT_NAME": resources_dict.get("SnapshotName"), + "NON_DEFAULT_USER": resources_dict.get("SnapshotName") + } + + yield replacements + # teardown + service_cleanup.service_cleanup(resources_dict) + + +@pytest.fixture(params=loader.list_scenarios(scenarios_directory), ids=loader.idfn) +def scenario(request, input_replacements): """ Parameterized fixture Provides scenarios to execute Supports parallel execution of scenarios """ - s = request.param - yield s - runner.teardown(s) + scenario_file_path = request.param + scenario = loader.load_scenario(scenario_file_path, input_replacements) + yield scenario + runner.teardown(scenario) @service_marker From d8fb9933c8508c8bd32463bdbe3508876bdaee97 Mon Sep 17 00:00:00 2001 From: Kumar Gaurav Sharma Date: Mon, 12 Jul 2021 14:01:58 -0700 Subject: [PATCH 3/4] Support customResourceReference at Scenario level and ability to override it at Step level --- test/declarative_test_fwk/__init__.py | 12 +++++ test/declarative_test_fwk/helper.py | 4 +- test/declarative_test_fwk/loader.py | 9 ++-- test/declarative_test_fwk/model.py | 43 ++++++++++++++---- test/declarative_test_fwk/runner.py | 45 +++++++++++-------- ...rio1.yaml => cmd_basic_create_update.yaml} | 19 +++----- .../test_scenarios.py} | 10 ++++- 7 files changed, 95 insertions(+), 47 deletions(-) create mode 100644 test/declarative_test_fwk/__init__.py rename test/e2e/scenarios/{scenario1.yaml => cmd_basic_create_update.yaml} (73%) rename test/e2e/{declarative_tests.py => tests/test_scenarios.py} (88%) diff --git a/test/declarative_test_fwk/__init__.py b/test/declarative_test_fwk/__init__.py new file mode 100644 index 00000000..5d489e5d --- /dev/null +++ b/test/declarative_test_fwk/__init__.py @@ -0,0 +1,12 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. diff --git a/test/declarative_test_fwk/helper.py b/test/declarative_test_fwk/helper.py index 7a2ab3a1..771ff351 100644 --- a/test/declarative_test_fwk/helper.py +++ b/test/declarative_test_fwk/helper.py @@ -81,9 +81,11 @@ def delete(self, reference: k8s.CustomResourceReference): sleep(self.DEFAULT_WAIT_SECS) self.wait_for_delete(reference) # throws exception if wait fails - def assert_expectations(self, expectations: dict, reference: k8s.CustomResourceReference): + def assert_expectations(self, verb: str, input_data: dict, expectations: dict, reference: k8s.CustomResourceReference): """ Asserts custom resource reference against supplied expectations + :param verb: expectations after performing the verb (apply, patch, delete) + :param input_data: input data to verb :param expectations: expectations to assert :param reference: custom resource reference :return: None diff --git a/test/declarative_test_fwk/loader.py b/test/declarative_test_fwk/loader.py index 0c4dfe16..5ad0ea37 100644 --- a/test/declarative_test_fwk/loader.py +++ b/test/declarative_test_fwk/loader.py @@ -14,7 +14,7 @@ """Test Scenarios loader for Declarative tests framework for custom resources """ -from declarative_test_fwk import model, helper +from declarative_test_fwk import model import pytest import os from typing import Iterable @@ -34,11 +34,12 @@ def list_scenarios(scenarios_directory: Path) -> Iterable[Path]: scenario_file_full_path = join(scenarios_directory, scenario_file) if not isfile(scenario_file_full_path) or not scenario_file.endswith(".yaml"): continue - scenarios_list.append(Path(scenario_file_full_path)) + scenario = load_scenario(Path(scenario_file_full_path)) + scenarios_list.append(pytest.param(Path(scenario_file_full_path),marks=marks(scenario))) return scenarios_list -def load_scenario(scenario_file: Path, replacements: dict = {}) -> Iterable: +def load_scenario(scenario_file: Path, replacements: dict = {}) -> model.Scenario: """ Loads scenario from given scenario_file :param scenario_file: yaml file containing scenarios @@ -50,7 +51,7 @@ def load_scenario(scenario_file: Path, replacements: dict = {}) -> Iterable: replacements["RANDOM_SUFFIX"] = random_suffix_name("", 32) scenario = model.Scenario(load_resource_file( scenario_file.parent, scenario_name, additional_replacements=replacements), replacements) - yield pytest.param(scenario, marks=marks(scenario)) + return scenario def idfn(scenario_file_full_path: Path) -> str: diff --git a/test/declarative_test_fwk/model.py b/test/declarative_test_fwk/model.py index 38fe7c2f..4cefb130 100644 --- a/test/declarative_test_fwk/model.py +++ b/test/declarative_test_fwk/model.py @@ -21,31 +21,40 @@ class Step: """ Represents a declarative test step """ - indent = "\t\t" - def __init__(self, config: dict, replacements: dict = {}): + def __init__(self, config: dict, custom_resource_details: dict, replacements: dict = {}): self.config = config + self.custom_resource_details = custom_resource_details self.replacements = replacements self.verb = None - self.input_data = None + self.input_data = {} self.expectations = None - self.resource_helper = None # (k8s.CustomResourceReference, ko) to teardown self.teardown_list = [] - supported_verbs=["create", "patch", "delete"] + supported_verbs = ["create", "patch", "delete"] for verb in supported_verbs: if verb not in self.config: continue self.verb = verb self.input_data = self.config.get(verb) + if type(self.input_data) is str: + # consider the input as resource name + # confirm that self.custom_resource_details must be provided with same name + if self.custom_resource_details["metadata"]["name"] != self.input_data: + raise ValueError(f"Unable to determine input data for '{self.verb}' at step: {self.id()}") + # self.custom_resource_details will be mixed in into self.input_data + self.input_data = {} break - if self.input_data: - self.expectations = self.config.get("expect") - self.resource_helper = helper.get_resource_helper(self.input_data.get("kind")) + if len(self.input_data) == 0 and not self.custom_resource_details: + raise ValueError(f"Unable to determine custom resource at step: {self.id()}") + + if self.custom_resource_details: + self.input_data = {**self.custom_resource_details, **self.input_data} + self.expectations = self.config.get("expect") def id(self) -> str: return self.config.get("id", "") @@ -53,6 +62,15 @@ def id(self) -> str: def description(self) -> str: return self.config.get("description", "") + def resource_kind(self) -> str: + return self.input_data.get("kind") + + def __str__(self) -> str: + return f"Step(id='{self.id()}')" + + def __repr__(self) -> str: + return str(self) + class Scenario: """ @@ -63,8 +81,9 @@ def __init__(self, config: dict, replacements: dict = {}): self.config = config self.test_steps = [] self.replacements = replacements + custom_resource_details = self.config.get("customResourceReference", {}) for step in self.config.get("steps", []): - self.test_steps.append(Step(step, replacements)) + self.test_steps.append(Step(step, custom_resource_details.copy(), replacements)) def id(self) -> str: return self.config.get("id", "") @@ -80,3 +99,9 @@ def marks(self) -> list: def steps(self): return self.test_steps + + def __str__(self) -> str: + return f"Scenario(id='{self.id()}')" + + def __repr__(self) -> str: + return str(self) diff --git a/test/declarative_test_fwk/runner.py b/test/declarative_test_fwk/runner.py index 6956e9bc..8680539e 100644 --- a/test/declarative_test_fwk/runner.py +++ b/test/declarative_test_fwk/runner.py @@ -14,7 +14,7 @@ """Runner for Declarative tests framework scenarios for custom resources """ -from declarative_test_fwk import model +from declarative_test_fwk import model, helper import pytest import sys import logging @@ -28,7 +28,7 @@ def run(scenario: model.Scenario) -> None: if not scenario: return - logging.info(f"Execute: Scenario: {scenario.id()}") + logging.info(f"Execute: {scenario}") for step in scenario.steps(): run_step(step) @@ -40,20 +40,20 @@ def teardown(scenario: model.Scenario) -> None: if not scenario: return - logging.info(f"Teardown: Scenario: {scenario.id()}") + logging.info(f"Teardown: {scenario}") teardown_failures = [] # tear down steps in reverse order for step in reversed(scenario.steps()): try: teardown_step(step) except: - error = f"Failed to teardown Step: {step.id()}. " \ + error = f"Failed to teardown: {step}. " \ f"Unexpected error: {sys.exc_info()[0]}" teardown_failures.append(error) logging.debug(error) if len(teardown_failures) != 0: - teardown_failures.insert(0, f"Failures during teardown. Scenario: {scenario.id()}") + teardown_failures.insert(0, f"Failures during teardown: {scenario}") failures = "\n\t- ".join(teardown_failures) logging.error(failures) pytest.fail(failures) @@ -68,54 +68,61 @@ def run_step(step: model.Step) -> None: if not step.verb: logging.warning( - f"skipped: Step: {step.id()}. No matching verb found." + f"skipped: {step}. No matching verb found." f" Supported verbs: create, patch, delete.") return + logging.info(f"Execute: {step}") if step.verb == "create": create_resource(step) elif step.verb == "patch": patch_resource(step) elif step.verb == "delete": - pass + delete_resource(step) assert_expectations(step) def create_resource(step: model.Step) -> None: - logging.debug(f"create: Step: {step.id()}") + logging.debug(f"create: {step}") if not step.input_data: return - - (reference, ko) = step.resource_helper.create(step.input_data, step.replacements) + resource_helper = helper.get_resource_helper(step.resource_kind()) + (reference, ko) = resource_helper.create(step.input_data, step.replacements) # track created reference to teardown later step.teardown_list.append((reference, ko)) def patch_resource(step: model.Step) -> None: - logging.debug(f"patch: Step: {step.id()}") + logging.debug(f"patch: {step}") if not step.input_data: return - (reference, ko) = step.resource_helper.patch(step.input_data, step.replacements) + resource_helper = helper.get_resource_helper(step.resource_kind()) + (reference, ko) = resource_helper.patch(step.input_data, step.replacements) # no need to teardown patched reference, its creator should tear it down. def delete_resource(step: model.Step, reference: k8s.CustomResourceReference = None) -> None: + resource_helper = helper.get_resource_helper(step.resource_kind()) if not reference: - logging.debug(f"delete: Step: {step.id()}") - reference = step.resource_helper.custom_resource_reference(step.input_data, step.replacements) + logging.debug(f"delete: {step}") + reference = resource_helper.custom_resource_reference(step.input_data, step.replacements) - step.resource_helper.delete(reference) + resource_helper.delete(reference) def assert_expectations(step: model.Step) -> None: - logging.debug(f"assert: Step: {step.id()}") + logging.info(f"assert: {step}") if not step.expectations: return - resource_helper = step.resource_helper + resource_helper = helper.get_resource_helper(step.resource_kind()) reference = resource_helper.custom_resource_reference(step.input_data, step.replacements) - resource_helper.assert_expectations(step.expectations, reference) + try: + resource_helper.assert_expectations(step.verb, step.input_data, step.expectations, reference) + except AssertionError as ae: + logging.error(f"AssertionError at {step}") + raise ae def teardown_step(step: model.Step) -> None: @@ -125,7 +132,7 @@ def teardown_step(step: model.Step) -> None: if not step or len(step.teardown_list) == 0: return - logging.info(f"teardown: Step: {step.id()}") + logging.info(f"teardown: {step}") for (reference, _) in step.teardown_list: if reference: diff --git a/test/e2e/scenarios/scenario1.yaml b/test/e2e/scenarios/cmd_basic_create_update.yaml similarity index 73% rename from test/e2e/scenarios/scenario1.yaml rename to test/e2e/scenarios/cmd_basic_create_update.yaml index ce9cff36..1a38d5e1 100644 --- a/test/e2e/scenarios/scenario1.yaml +++ b/test/e2e/scenarios/cmd_basic_create_update.yaml @@ -6,14 +6,15 @@ usecases: #marks: # - slow # - blocked +customResourceReference: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: ReplicationGroup + metadata: + name: test$RANDOM_SUFFIX steps: - id: "create_CMD_replication_group" description: "Initial config" create: - apiVersion: $CRD_GROUP/$CRD_VERSION - kind: ReplicationGroup - metadata: - name: test$RANDOM_SUFFIX spec: engine: redis replicationGroupID: test$RANDOM_SUFFIX @@ -28,10 +29,6 @@ steps: - id: "invalid_cacheNodeType_on_CMD_replication_group_causes_Terminal_condition" description: "Negative test include transitEncryptionEnabled" patch: - apiVersion: $CRD_GROUP/$CRD_VERSION - kind: ReplicationGroup - metadata: - name: test$RANDOM_SUFFIX spec: cacheNodeType: cache.micro # invalid value expect: @@ -40,8 +37,4 @@ steps: ACK.Terminal: "True" - id: "delete_CMD_RG" description: "Delete cluster mode disabled replication group" - delete: - apiVersion: $CRD_GROUP/$CRD_VERSION - kind: ReplicationGroup - metadata: - name: test$RANDOM_SUFFIX + delete: test$RANDOM_SUFFIX diff --git a/test/e2e/declarative_tests.py b/test/e2e/tests/test_scenarios.py similarity index 88% rename from test/e2e/declarative_tests.py rename to test/e2e/tests/test_scenarios.py index c27d1beb..b2b4fa06 100644 --- a/test/e2e/declarative_tests.py +++ b/test/e2e/tests/test_scenarios.py @@ -29,6 +29,14 @@ @helper.resource_helper("ReplicationGroup") class ReplicationGroupHelper(helper.ResourceHelper): + def assert_expectations(self, verb: str, input_data: dict, expectations: dict, + reference: k8s.CustomResourceReference): + # default assertions + super().assert_expectations(verb, input_data, expectations, reference) + + # perform custom server side checks based on: + # verb, input data to verb, expectations for given resource + """ Helper for replication group scenarios """ @@ -83,7 +91,7 @@ def scenario(request, input_replacements): @service_marker -class TestSuite: +class TestScenarios: """ Declarative scenarios based test suite """ From be3a53aced97048c528efe75ef22197a36dbdab5 Mon Sep 17 00:00:00 2001 From: Kumar Gaurav Sharma Date: Thu, 29 Jul 2021 18:14:57 -0700 Subject: [PATCH 4/4] declarative test framework - resource from file, wait before assert expectations, addressed comments --- test/declarative_test_fwk/helper.py | 173 ------------ test/declarative_test_fwk/model.py | 107 ------- test/e2e/bootstrap_resources.py | 16 ++ test/e2e/conftest.py | 4 - .../declarative_test_fwk/__init__.py | 0 test/e2e/declarative_test_fwk/helper.py | 261 ++++++++++++++++++ test/{ => e2e}/declarative_test_fwk/loader.py | 79 +++--- test/e2e/declarative_test_fwk/model.py | 187 +++++++++++++ test/{ => e2e}/declarative_test_fwk/runner.py | 156 ++++++++--- .../scenarios/cmd_basic_create_update.yaml | 17 +- test/e2e/tests/test_scenarios.py | 50 ++-- 11 files changed, 660 insertions(+), 390 deletions(-) delete mode 100644 test/declarative_test_fwk/helper.py delete mode 100644 test/declarative_test_fwk/model.py rename test/{ => e2e}/declarative_test_fwk/__init__.py (100%) create mode 100644 test/e2e/declarative_test_fwk/helper.py rename test/{ => e2e}/declarative_test_fwk/loader.py (51%) create mode 100644 test/e2e/declarative_test_fwk/model.py rename test/{ => e2e}/declarative_test_fwk/runner.py (54%) diff --git a/test/declarative_test_fwk/helper.py b/test/declarative_test_fwk/helper.py deleted file mode 100644 index 771ff351..00000000 --- a/test/declarative_test_fwk/helper.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may -# not use this file except in compliance with the License. A copy of the -# License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -"""Helper for Declarative tests framework for custom resources -""" - -import logging -from time import sleep -from acktest.k8s import resource as k8s - -TEST_HELPERS = dict() - - -def resource_helper(resource_kind: str): - """ - Decorator to discover Custom Resource Helper - :param resource_kind: custom resource kind - """ - def registrar(cls): - TEST_HELPERS[resource_kind.lower()] = cls - logging.info(f"Registered ResourceHepler: {cls.__name__} for custom resource kind: {resource_kind}") - return registrar - - -class ResourceHelper: - """ - Provides generic verb (create, patch, delete) methods for custom resources. - Keep its methods stateless. Methods are on instance to allow specialization. - """ - DEFAULT_WAIT_SECS = 30 - - def create(self, input_data: dict, input_replacements: dict = {}): - """ - Creates custom resource - :param input_data: resource details - :param input_replacements: input replacements - :return: k8s.CustomResourceReference, created custom resource - """ - reference = self.custom_resource_reference(input_data, input_replacements) - _ = k8s.create_custom_resource(reference, input_data) - resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) - assert resource is not None - return (reference, resource) - - def patch(self, input_data: dict, input_replacements: dict = {}): - """ - Patches custom resource - :param input_data: resource patch details - :param input_replacements: input replacements - :return: k8s.CustomResourceReference, created custom resource - """ - reference = self.custom_resource_reference(input_data, input_replacements) - _ = k8s.patch_custom_resource(reference, input_data) - sleep(self.DEFAULT_WAIT_SECS) # required as controller has likely not placed the resource in modifying - resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) - assert resource is not None - return (reference, resource) - - def delete(self, reference: k8s.CustomResourceReference): - """ - Deletes custom resource and waits for delete completion - :param reference: resource reference - :return: None - """ - resource = k8s.get_resource(reference) - if not resource: - logging.warning(f"ResourceReference {reference} not found. Not invoking k8s delete api.") - return - - k8s.delete_custom_resource(reference) - sleep(self.DEFAULT_WAIT_SECS) - self.wait_for_delete(reference) # throws exception if wait fails - - def assert_expectations(self, verb: str, input_data: dict, expectations: dict, reference: k8s.CustomResourceReference): - """ - Asserts custom resource reference against supplied expectations - :param verb: expectations after performing the verb (apply, patch, delete) - :param input_data: input data to verb - :param expectations: expectations to assert - :param reference: custom resource reference - :return: None - """ - # condition assertion contains wait logic - self._assert_conditions(expectations, reference) - - # conditions expectations met, now check current resource against expectations - resource = k8s.get_resource(reference) - self.assert_items(expectations.get("status"), resource.get("status")) - - # self._assert_state(expectations.get("spec"), resource) # uncomment to support spec assertions - - def _assert_conditions(self, expectations: dict, reference: k8s.CustomResourceReference): - expect_conditions: dict = {} - if "status" in expectations and "conditions" in expectations["status"]: - expect_conditions = expectations["status"]["conditions"] - - for (condition_name, condition_value) in expect_conditions.items(): - assert k8s.wait_on_condition(reference, condition_name, condition_value, wait_periods=30) - - def assert_items(self, expectations: dict, state: dict): - """ - Asserts state against supplied expectations - Override it as needed for custom verifications - :param expectations: dictionary with items to assert in state - :param state: dictionary with items - :return: None - """ - if not expectations: - return - if not state: - assert expectations == state - - for (property, value) in expectations.items(): - # conditions are processed separately - if property == "conditions": - continue - assert (property, value) == (property, state.get(property)) - - def custom_resource_reference(self, input_data: dict, input_replacements: dict = {}) -> k8s.CustomResourceReference: - """ - Helper method to provide k8s.CustomResourceReference for supplied input - :param input_data: custom resource input data - :param input_replacements: input replacements - :return: k8s.CustomResourceReference - """ - resource_plural = self.resource_plural(input_data.get("kind")) - resource_name = input_data.get("metadata").get("name") - crd_group = input_replacements.get("CRD_GROUP") - crd_version = input_replacements.get("CRD_VERSION") - - reference = k8s.CustomResourceReference( - crd_group, crd_version, resource_plural, resource_name, namespace="default") - return reference - - def wait_for_delete(self, reference: k8s.CustomResourceReference): - """ - Override this method to implement custom resource delete logic. - :param reference: custom resource reference - :return: None - """ - logging.debug(f"No-op wait_for_delete()") - - def resource_plural(self, resource_kind: str) -> str: - """ - Provide plural string for supplied custom resource kind - Override as needed - :param resource_kind: custom resource kind - :return: plural string - """ - return resource_kind.lower() + "s" - - -def get_resource_helper(resource_kind: str) -> ResourceHelper: - """ - Provides ResourceHelper for supplied custom resource kind - If no helper is registered for the supplied resource kind then returns default ResourceHelper - :param resource_kind: custom resource kind - :return: custom resource helper - """ - helper_cls = TEST_HELPERS.get(resource_kind.lower()) - if helper_cls: - return helper_cls() - return ResourceHelper() diff --git a/test/declarative_test_fwk/model.py b/test/declarative_test_fwk/model.py deleted file mode 100644 index 4cefb130..00000000 --- a/test/declarative_test_fwk/model.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may -# not use this file except in compliance with the License. A copy of the -# License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -"""Model for Declarative tests framework for custom resources -""" - -from declarative_test_fwk import helper - - -class Step: - """ - Represents a declarative test step - """ - - def __init__(self, config: dict, custom_resource_details: dict, replacements: dict = {}): - self.config = config - self.custom_resource_details = custom_resource_details - self.replacements = replacements - - self.verb = None - self.input_data = {} - self.expectations = None - - # (k8s.CustomResourceReference, ko) to teardown - self.teardown_list = [] - - supported_verbs = ["create", "patch", "delete"] - for verb in supported_verbs: - if verb not in self.config: - continue - self.verb = verb - self.input_data = self.config.get(verb) - if type(self.input_data) is str: - # consider the input as resource name - # confirm that self.custom_resource_details must be provided with same name - if self.custom_resource_details["metadata"]["name"] != self.input_data: - raise ValueError(f"Unable to determine input data for '{self.verb}' at step: {self.id()}") - # self.custom_resource_details will be mixed in into self.input_data - self.input_data = {} - break - - if len(self.input_data) == 0 and not self.custom_resource_details: - raise ValueError(f"Unable to determine custom resource at step: {self.id()}") - - if self.custom_resource_details: - self.input_data = {**self.custom_resource_details, **self.input_data} - self.expectations = self.config.get("expect") - - def id(self) -> str: - return self.config.get("id", "") - - def description(self) -> str: - return self.config.get("description", "") - - def resource_kind(self) -> str: - return self.input_data.get("kind") - - def __str__(self) -> str: - return f"Step(id='{self.id()}')" - - def __repr__(self) -> str: - return str(self) - - -class Scenario: - """ - Represents a declarative test scenario with steps - """ - - def __init__(self, config: dict, replacements: dict = {}): - self.config = config - self.test_steps = [] - self.replacements = replacements - custom_resource_details = self.config.get("customResourceReference", {}) - for step in self.config.get("steps", []): - self.test_steps.append(Step(step, custom_resource_details.copy(), replacements)) - - def id(self) -> str: - return self.config.get("id", "") - - def description(self) -> str: - return self.config.get("description", "") - - def usecases(self) -> list: - return self.config.get("usecases", []) - - def marks(self) -> list: - return self.config.get("marks", []) - - def steps(self): - return self.test_steps - - def __str__(self) -> str: - return f"Scenario(id='{self.id()}')" - - def __repr__(self) -> str: - return str(self) diff --git a/test/e2e/bootstrap_resources.py b/test/e2e/bootstrap_resources.py index 9572fa75..205b8fd9 100644 --- a/test/e2e/bootstrap_resources.py +++ b/test/e2e/bootstrap_resources.py @@ -34,6 +34,22 @@ class TestBootstrapResources: CWLogGroup2: str CPGName: str + def replacement_dict(self): + return { + "SNS_TOPIC_ARN": self.SnsTopic1, + "SNS_TOPIC_ARN_2": self.SnsTopic2, + "SG_ID": self.SecurityGroup1, + "SG_ID_2": self.SecurityGroup2, + "USERGROUP_ID": self.UserGroup1, + "USERGROUP_ID_2": self.UserGroup2, + "KMS_KEY_ID": self.KmsKeyID, + "SNAPSHOT_NAME": self.SnapshotName, + "NON_DEFAULT_USER": self.NonDefaultUser, + "LOG_GROUP": self.CWLogGroup1, + "LOG_GROUP_2": self.CWLogGroup2, + "CACHE_PARAMETER_GROUP_NAME": self.CPGName + } + _bootstrap_resources = None diff --git a/test/e2e/conftest.py b/test/e2e/conftest.py index 25107c74..8a1ce7ab 100644 --- a/test/e2e/conftest.py +++ b/test/e2e/conftest.py @@ -27,9 +27,6 @@ def pytest_configure(config): config.addinivalue_line( "markers", "service(arg): mark test associated with a given service" ) - config.addinivalue_line( - "markers", "usecase(arg): mark test associated with a given usecase" - ) config.addinivalue_line( "markers", "slow: mark test as slow to run" ) @@ -49,7 +46,6 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip_slow) if "blocked" in item.keywords and not config.getoption("--runblocked"): item.add_marker(skip_blocked) - # TODO: choose test per 'usecase' selector # Provide a k8s client to interact with the integration test cluster diff --git a/test/declarative_test_fwk/__init__.py b/test/e2e/declarative_test_fwk/__init__.py similarity index 100% rename from test/declarative_test_fwk/__init__.py rename to test/e2e/declarative_test_fwk/__init__.py diff --git a/test/e2e/declarative_test_fwk/helper.py b/test/e2e/declarative_test_fwk/helper.py new file mode 100644 index 00000000..7705f38e --- /dev/null +++ b/test/e2e/declarative_test_fwk/helper.py @@ -0,0 +1,261 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""Helper for Declarative tests framework for custom resources +""" + +from e2e.declarative_test_fwk import model + +import logging +from typing import Tuple +from time import sleep +from acktest.k8s import resource as k8s + +# holds custom resource helper references +TEST_HELPERS = dict() + + +def register_resource_helper(resource_kind: str, resource_plural: str): + """Decorator to discover Custom Resource Helper + + Args: + resource_kind: custom resource kind + resource_plural: custom resource kind plural + + Returns: + wrapper + """ + + def registrar(cls): + global TEST_HELPERS + if issubclass(cls, ResourceHelper): + TEST_HELPERS[resource_kind.lower()] = cls + cls.resource_plural = resource_plural.lower() + logging.info(f"Registered ResourceHelper: {cls.__name__} for custom resource kind: {resource_kind}") + else: + msg = f"Unable to register helper for {resource_kind} resource: {cls} is not a subclass of ResourceHelper" + logging.error(msg) + raise Exception(msg) + return registrar + + +class ResourceHelper: + """Provides generic verb (create, patch, delete) methods for custom resources. + Keep its methods stateless. Methods are on instance to allow specialization. + """ + + DEFAULT_WAIT_SECS = 30 + + def create(self, input_data: dict, input_replacements: dict = {}) -> Tuple[k8s.CustomResourceReference, dict]: + """Creates custom resource inside Kubernetes cluster per the specifications in input data. + + Args: + input_data: custom resource details + input_replacements: input replacements + + Returns: + k8s.CustomResourceReference, created custom resource + """ + + reference = self.custom_resource_reference(input_data, input_replacements) + _ = k8s.create_custom_resource(reference, input_data) + resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) + assert resource is not None + return reference, resource + + def patch(self, input_data: dict, input_replacements: dict = {}) -> Tuple[k8s.CustomResourceReference, dict]: + """Patches custom resource inside Kubernetes cluster per the specifications in input data. + + Args: + input_data: custom resource patch details + input_replacements: input replacements + + Returns: + k8s.CustomResourceReference, created custom resource + """ + + reference = self.custom_resource_reference(input_data, input_replacements) + _ = k8s.patch_custom_resource(reference, input_data) + sleep(self.DEFAULT_WAIT_SECS) # required as controller has likely not placed the resource in modifying + resource = k8s.wait_resource_consumed_by_controller(reference, wait_periods=10) + assert resource is not None + return reference, resource + + def delete(self, reference: k8s.CustomResourceReference) -> None: + """Deletes custom resource inside Kubernetes cluster and waits for delete completion + + Args: + reference: custom resource reference + + Returns: + None + """ + + resource = k8s.get_resource(reference) + if not resource: + logging.warning(f"ResourceReference {reference} not found. Not invoking k8s delete api.") + return + + k8s.delete_custom_resource(reference, wait_periods=30, period_length=60) # throws exception if wait fails + sleep(self.DEFAULT_WAIT_SECS) + self.wait_for_delete(reference) # throws exception if wait fails + + def assert_expectations(self, verb: str, input_data: dict, expectations: model.ExpectDict, reference: k8s.CustomResourceReference) -> None: + """Asserts custom resource reference inside Kubernetes cluster against the supplied expectations + + :param verb: expectations after performing the verb (apply, patch, delete) + :param input_data: input data to verb + :param expectations: expectations to assert + :param reference: custom resource reference + :return: None + """ + self._assert_conditions(expectations, reference, wait=False) + # conditions expectations met, now check current resource against expectations + resource = k8s.get_resource(reference) + self.assert_items(expectations.get("status"), resource.get("status")) + + # self._assert_state(expectations.get("spec"), resource) # uncomment to support spec assertions + + def wait_for(self, wait_expectations: dict, reference: k8s.CustomResourceReference) -> None: + """Waits for custom resource reference details inside Kubernetes cluster to match supplied config, + currently supports wait on "status.conditions", + it can be enhanced later for wait on any/other properties. + + Args: + wait_expectations: properties to wait for + reference: custom resource reference + + Returns: + None + """ + + # wait for conditions + self._assert_conditions(wait_expectations, reference, wait=True) + + def _assert_conditions(self, expectations: dict, reference: k8s.CustomResourceReference, wait: bool = True) -> None: + expect_conditions: dict = {} + if "status" in expectations and "conditions" in expectations["status"]: + expect_conditions = expectations["status"]["conditions"] + + default_wait_periods = 60 + # period_length = 1 will result in condition check every second + default_period_length = 1 + for (condition_name, expected_value) in expect_conditions.items(): + if type(expected_value) is str: + # Example: ACK.Terminal: "True" + if wait: + assert k8s.wait_on_condition(reference, condition_name, expected_value, + wait_periods=default_wait_periods, period_length=default_period_length) + else: + actual_condition = k8s.get_resource_condition(reference, condition_name) + assert actual_condition is not None + assert expected_value == actual_condition.get("status"), f"Condition status mismatch. Expected condition: {condition_name} - {expected_value} but found {actual_condition}" + + elif type(expected_value) is dict: + # Example: + # ACK.ResourceSynced: + # status: "False" + # message: "Expected message ..." + # timeout: 60 # seconds + condition_value = expected_value.get("status") + condition_message = expected_value.get("message") + # default wait 60 seconds + wait_timeout = expected_value.get("timeout", default_wait_periods) + + if wait: + assert k8s.wait_on_condition(reference, condition_name, condition_value, + wait_periods=wait_timeout, period_length=default_period_length) + + actual_condition = k8s.get_resource_condition(reference, + condition_name) + assert actual_condition is not None + assert condition_value == actual_condition.get("status"), f"Condition status mismatch. Expected condition: {condition_name} - {expected_value} but found {actual_condition}" + if condition_message is not None: + assert condition_message == actual_condition.get("message"), f"Condition message mismatch. Expected condition: {condition_name} - {expected_value} but found {actual_condition}" + + else: + raise Exception(f"Condition {condition_name} is provided with invalid value: {expected_value} ") + + def assert_items(self, expectations: dict, state: dict) -> None: + """Asserts state against supplied expectations + Override it as needed for custom verifications + + Args: + expectations: dictionary with items (expected) to assert in state + state: dictionary with items (actual) + + Returns: + None + """ + + if not expectations: + # nothing to assert as there are no expectations + return + if not state: + # there are expectations but no given state to validate + # following assert will fail and assert introspection will provide useful information for debugging + assert expectations == state + + for (key, value) in expectations.items(): + # conditions are processed separately + if key == "conditions": + continue + assert (key, value) == (key, state.get(key)) + + def custom_resource_reference(self, input_data: dict, input_replacements: dict = {}) -> k8s.CustomResourceReference: + """Helper method to provide k8s.CustomResourceReference for supplied input + + Args: + input_data: custom resource input data + input_replacements: input replacements + + Returns: + k8s.CustomResourceReference + """ + + resource_name = input_data.get("metadata").get("name") + crd_group = input_replacements.get("CRD_GROUP") + crd_version = input_replacements.get("CRD_VERSION") + + reference = k8s.CustomResourceReference( + crd_group, crd_version, self.resource_plural, resource_name, namespace="default") + return reference + + def wait_for_delete(self, reference: k8s.CustomResourceReference) -> None: + """Override this method to implement custom wail logic on resource delete. + + Args: + reference: custom resource reference + + Returns: + None + """ + + logging.debug(f"No-op wait_for_delete()") + + +def get_resource_helper(resource_kind: str) -> ResourceHelper: + """Provides ResourceHelper for supplied custom resource kind + If no helper is registered for the supplied resource kind then returns default ResourceHelper + + Args: + resource_kind: custom resource kind string + + Returns: + custom resource helper instance + """ + + helper_cls = TEST_HELPERS.get(resource_kind.lower()) + if helper_cls: + return helper_cls() + return ResourceHelper() diff --git a/test/declarative_test_fwk/loader.py b/test/e2e/declarative_test_fwk/loader.py similarity index 51% rename from test/declarative_test_fwk/loader.py rename to test/e2e/declarative_test_fwk/loader.py index 5ad0ea37..e6fd3cd2 100644 --- a/test/declarative_test_fwk/loader.py +++ b/test/e2e/declarative_test_fwk/loader.py @@ -14,65 +14,82 @@ """Test Scenarios loader for Declarative tests framework for custom resources """ -from declarative_test_fwk import model +from e2e.declarative_test_fwk import model import pytest import os -from typing import Iterable +from typing import Iterable, List from pathlib import Path from os.path import isfile, join from acktest.resources import load_resource_file, random_suffix_name -def list_scenarios(scenarios_directory: Path) -> Iterable[Path]: - """ - Lists test scenarios from given directory - :param scenarios_directory: directory containing scenarios yaml files - :return: Iterable scenarios +def list_scenarios(scenarios_directory: Path) -> Iterable: + """Lists test scenarios from given directory + + Args: + scenarios_directory: directory containing scenarios yaml files + + Returns: + Iterable scenarios for pytest parameterized fixture """ + scenarios_list = [] - for scenario_file in os.listdir(scenarios_directory): + for scenario_file in sorted(os.listdir(scenarios_directory)): scenario_file_full_path = join(scenarios_directory, scenario_file) if not isfile(scenario_file_full_path) or not scenario_file.endswith(".yaml"): continue - scenario = load_scenario(Path(scenario_file_full_path)) - scenarios_list.append(pytest.param(Path(scenario_file_full_path),marks=marks(scenario))) + scenarios_list.append(pytest.param(Path(scenario_file_full_path), marks=marks(Path(scenario_file_full_path)))) return scenarios_list -def load_scenario(scenario_file: Path, replacements: dict = {}) -> model.Scenario: - """ - Loads scenario from given scenario_file - :param scenario_file: yaml file containing scenarios - :param replacements: input replacements - :return: Iterable scenarios +def load_scenario(scenario_file: Path, resource_directory: Path = None, replacements: dict = {}) -> model.Scenario: + """Loads scenario from given scenario_file + + Args: + scenario_file: yaml file containing scenario + resource_directory: Path to custom resources directory + replacements: input replacements + + Returns: + Scenario reference """ - scenario_name = scenario_file.name.split(".yaml")[0] + + scenario_name = scenario_file.stem replacements = replacements.copy() - replacements["RANDOM_SUFFIX"] = random_suffix_name("", 32) - scenario = model.Scenario(load_resource_file( + replacements["RANDOM_SUFFIX"] = random_suffix_name("", 16) + scenario = model.Scenario(resource_directory, load_resource_file( scenario_file.parent, scenario_name, additional_replacements=replacements), replacements) return scenario def idfn(scenario_file_full_path: Path) -> str: + """Provides scenario file name as scenario test id + + Args: + scenario_file_full_path: test scenario file path + + Returns: + scenario test id string """ - Provides scenario file name as scenario test id - :param scenario: test scenario file path - :return: scenario test id string - """ + return scenario_file_full_path.name -def marks(scenario: model.Scenario) -> list: - """ - Provides pytest markers for the scenario - :param scenario: test scenario - :return: markers for the scenario +def marks(scenario_file_path: Path) -> List: + """Provides pytest markers for the given scenario + + Args: + scenario_file_path: test scenario file path + + Returns: + pytest markers for the scenario """ + + scenario_config = load_resource_file( + scenario_file_path.parent, scenario_file_path.stem) + markers = [] - for usecase in scenario.usecases(): - markers.append(pytest.mark.usecase(arg=usecase)) - for mark in scenario.marks(): + for mark in scenario_config.get("marks", []): if mark == "canary": markers.append(pytest.mark.canary) elif mark == "slow": diff --git a/test/e2e/declarative_test_fwk/model.py b/test/e2e/declarative_test_fwk/model.py new file mode 100644 index 00000000..9940edeb --- /dev/null +++ b/test/e2e/declarative_test_fwk/model.py @@ -0,0 +1,187 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""Model for Declarative tests framework for custom resources +""" + + +from enum import Enum, auto +from typing import TypedDict, Dict, Union, List +from pathlib import Path +from os.path import join +from acktest.resources import load_resource_file + + +class Verb(Enum): + """ + Verb for custom resource in a test step. + """ + create = auto() + patch = auto() + delete = auto() + + +# fields for 'resource' field in a test Scenario +class ResourceDict(TypedDict, total=False): + apiVersion: str + kind: str + metadata: Dict + + +# fields for 'create' Verb in a test step +class CreateDict(ResourceDict): + spec: Dict + + +# fields for 'patch' Verb in a test step +class PatchDict(ResourceDict): + spec: Dict + + +# fields for 'delete' Verb in a test step +class DeleteDict(ResourceDict): + pass + + +# fields for 'expect' field in a test step +class ExpectDict(TypedDict, total=False): + spec: Dict + status: Dict + + +# fields in a test step +class StepDict(TypedDict, total=False): + id: str + description: str + create: Union[str, CreateDict] + patch: Union[str, PatchDict] + delete: Union[str, DeleteDict] + wait: Union[int, Dict] + expect: ExpectDict + + +class Step: + """ + Represents a declarative test step + """ + + def __init__(self, resource_directory: Path, config: StepDict, custom_resource_details: dict, replacements: dict = {}): + self.config = config + self.custom_resource_details = custom_resource_details + self.replacements = replacements + + self.verb = None + self.input_data = {} + self.expectations: ExpectDict = None + + # (k8s.CustomResourceReference, ko) to teardown + self.teardown_list = [] + + # validate: only one verb per step + step_verb = None + for verb in list(Verb): + if verb.name in self.config: + if not step_verb: + step_verb = verb + else: + raise ValueError(f"Multiple verbs specified for step: {self.id}." + f" Please specify only one verb from" + f" supported verbs: { {verb.name for verb in list(Verb)} }.") + + # a step with no verb can be used to assert preconditions + # thus, verb is optional. + if step_verb: + self.verb = step_verb + self.input_data = self.config.get(step_verb.name) + if type(self.input_data) is str: + if self.input_data.endswith(".yaml"): + # load input data from resource file + resource_file_name = self.input_data + resource_file_path = Path(join(resource_directory, resource_file_name)) + self.input_data = load_resource_file( + resource_file_path.parent, resource_file_path.stem, additional_replacements=replacements) + else: + # consider the input as resource name string + # confirm that self.custom_resource_details must be provided with same name + if self.custom_resource_details["metadata"]["name"] != self.input_data: + raise ValueError(f"Unable to determine input data for '{self.verb}' at step: {self.id}") + # self.custom_resource_details will be mixed in into self.input_data + self.input_data = {} + + if len(self.input_data) == 0 and not self.custom_resource_details: + raise ValueError(f"Unable to determine custom resource at step: {self.id}") + + if self.custom_resource_details: + self.input_data = {**self.custom_resource_details, **self.input_data} + + self.wait = self.config.get("wait") + self.expectations = self.config.get("expect") + + @property + def id(self) -> str: + return self.config.get("id", "") + + @property + def description(self) -> str: + return self.config.get("description", "") + + @property + def resource_kind(self) -> str: + return self.input_data.get("kind") + + def __str__(self) -> str: + return f"Step(id='{self.id}')" + + def __repr__(self) -> str: + return str(self) + + +# fields in a test scenario +class ScenarioDict(TypedDict, total=False): + id: str + description: str + marks: List[str] + resource: ResourceDict + steps: List[StepDict] + + +class Scenario: + """ + Represents a declarative test scenario with steps + """ + + def __init__(self, resource_directory: Path, config: ScenarioDict, replacements: dict = {}): + self.config = config + self.test_steps = [] + self.replacements = replacements + custom_resource_details = self.config.get("resource", {}) + for step_config in self.config.get("steps", []): + self.test_steps.append(Step(resource_directory, step_config, custom_resource_details.copy(), replacements)) + + @property + def id(self) -> str: + return self.config.get("id", "") + + @property + def description(self) -> str: + return self.config.get("description", "") + + @property + def steps(self): + return self.test_steps + + def __str__(self) -> str: + return f"Scenario(id='{self.id}')" + + def __repr__(self) -> str: + return str(self) diff --git a/test/declarative_test_fwk/runner.py b/test/e2e/declarative_test_fwk/runner.py similarity index 54% rename from test/declarative_test_fwk/runner.py rename to test/e2e/declarative_test_fwk/runner.py index 8680539e..4492c13e 100644 --- a/test/declarative_test_fwk/runner.py +++ b/test/e2e/declarative_test_fwk/runner.py @@ -14,43 +14,49 @@ """Runner for Declarative tests framework scenarios for custom resources """ -from declarative_test_fwk import model, helper +from e2e.declarative_test_fwk import model, helper import pytest import sys import logging +from time import sleep from acktest.k8s import resource as k8s def run(scenario: model.Scenario) -> None: + """Runs steps in the given scenario + + Args: + scenario: the scenario to run + + Returns: + None """ - Runs steps in the scenario - """ - if not scenario: - return logging.info(f"Execute: {scenario}") - for step in scenario.steps(): + for step in scenario.steps: run_step(step) def teardown(scenario: model.Scenario) -> None: + """Teardown steps in the given scenario in reverse run order + + Args: + scenario: the scenario to teardown + + Returns: + None """ - Teardown steps in the scenario in reverse order - """ - if not scenario: - return logging.info(f"Teardown: {scenario}") teardown_failures = [] # tear down steps in reverse order - for step in reversed(scenario.steps()): + for step in reversed(scenario.steps): try: teardown_step(step) except: error = f"Failed to teardown: {step}. " \ f"Unexpected error: {sys.exc_info()[0]}" teardown_failures.append(error) - logging.debug(error) if len(teardown_failures) != 0: teardown_failures.insert(0, f"Failures during teardown: {scenario}") @@ -60,63 +66,137 @@ def teardown(scenario: model.Scenario) -> None: def run_step(step: model.Step) -> None: - """ - Runs step - """ - if not step: - return + """Runs a test scenario step - if not step.verb: - logging.warning( - f"skipped: {step}. No matching verb found." - f" Supported verbs: create, patch, delete.") - return + Args: + step: the step to run + + Returns: + None + """ logging.info(f"Execute: {step}") - if step.verb == "create": + if step.verb == model.Verb.create: create_resource(step) - elif step.verb == "patch": + elif step.verb == model.Verb.patch: patch_resource(step) - elif step.verb == "delete": + elif step.verb == model.Verb.delete: delete_resource(step) + wait(step) assert_expectations(step) def create_resource(step: model.Step) -> None: - logging.debug(f"create: {step}") + """Perform the Verb "create" for given test step. + It results in creating custom resource inside Kubernetes cluster per the specification from the step. + + Args: + step: test step + + Returns: + None + """ + + logging.debug(f"create: {step}") if not step.input_data: return - resource_helper = helper.get_resource_helper(step.resource_kind()) + resource_helper = helper.get_resource_helper(step.resource_kind) (reference, ko) = resource_helper.create(step.input_data, step.replacements) # track created reference to teardown later step.teardown_list.append((reference, ko)) def patch_resource(step: model.Step) -> None: - logging.debug(f"patch: {step}") + """Perform the Verb "patch" for given test step. + It results in patching custom resource inside Kubernetes cluster per the specification from the step. + + Args: + step: test step + + Returns: + None + """ + + logging.debug(f"patch: {step}") if not step.input_data: return - resource_helper = helper.get_resource_helper(step.resource_kind()) + resource_helper = helper.get_resource_helper(step.resource_kind) (reference, ko) = resource_helper.patch(step.input_data, step.replacements) # no need to teardown patched reference, its creator should tear it down. def delete_resource(step: model.Step, reference: k8s.CustomResourceReference = None) -> None: - resource_helper = helper.get_resource_helper(step.resource_kind()) + """Perform the Verb "delete" for given custom resource reference in given test step. + It results in deleting the custom resource inside Kubernetes cluster. + + Args: + step: test step + reference: custom resource reference to delete + + Returns: + None + """ + + resource_helper = helper.get_resource_helper(step.resource_kind) if not reference: logging.debug(f"delete: {step}") reference = resource_helper.custom_resource_reference(step.input_data, step.replacements) + if k8s.get_resource_exists(reference): + logging.debug(f"deleting resource: {reference}") + resource_helper.delete(reference) + else: + logging.info(f"Resource already deleted: {reference}") + + +def wait(step: model.Step) -> None: + """Performs wait logic for the given step. + The step provides the wait details (properties/conditions values to wait for) + + Args: + step: test step + + Returns: + None + """ - resource_helper.delete(reference) + logging.debug(f"wait: {step}") + if not step.wait: + return + + if type(step.wait) is int: + interval_seconds = step.wait + logging.debug(f"Going to sleep for {interval_seconds} seconds during step {step}") + sleep(interval_seconds) + return + + resource_helper = helper.get_resource_helper(step.resource_kind) + reference = resource_helper.custom_resource_reference(step.input_data, step.replacements) + try: + resource_helper.wait_for(step.wait, reference) + except AssertionError as ae: + logging.error(f"Wait failed, AssertionError at {step}") + raise ae + except Exception as e: + logging.error(f"Wait failed, Exception at {step}") + raise e def assert_expectations(step: model.Step) -> None: + """Asserts expectations as specified in the Step. + + Args: + step: test step + + Returns: + None + """ + logging.info(f"assert: {step}") if not step.expectations: return - resource_helper = helper.get_resource_helper(step.resource_kind()) + resource_helper = helper.get_resource_helper(step.resource_kind) reference = resource_helper.custom_resource_reference(step.input_data, step.replacements) try: resource_helper.assert_expectations(step.verb, step.input_data, step.expectations, reference) @@ -126,9 +206,15 @@ def assert_expectations(step: model.Step) -> None: def teardown_step(step: model.Step) -> None: + """Teardown custom resources that were created during step execution (run) inside Kubernetes cluster. + + Args: + step: test step + + Returns: + None """ - Teardown resources from the step - """ + if not step or len(step.teardown_list) == 0: return @@ -136,7 +222,7 @@ def teardown_step(step: model.Step) -> None: for (reference, _) in step.teardown_list: if reference: - delete_resource(reference) + delete_resource(step, reference) # clear list step.teardown_list = [] diff --git a/test/e2e/scenarios/cmd_basic_create_update.yaml b/test/e2e/scenarios/cmd_basic_create_update.yaml index 1a38d5e1..aed144c9 100644 --- a/test/e2e/scenarios/cmd_basic_create_update.yaml +++ b/test/e2e/scenarios/cmd_basic_create_update.yaml @@ -1,12 +1,9 @@ id: "RG_CMD_basic_create_update" description: "In this test we create RG with absolutely minimum configuration and try to update it." -usecases: - - replication_group - - cmd #marks: # - slow # - blocked -customResourceReference: +resource: apiVersion: $CRD_GROUP/$CRD_VERSION kind: ReplicationGroup metadata: @@ -21,20 +18,28 @@ steps: replicationGroupDescription: Basic create and update of CMD RG cacheNodeType: cache.t3.micro numNodeGroups: 1 - expect: + wait: status: conditions: - ACK.ResourceSynced: "True" + ACK.ResourceSynced: + status: "True" + timeout: 1800 + expect: + status: status: "available" - id: "invalid_cacheNodeType_on_CMD_replication_group_causes_Terminal_condition" description: "Negative test include transitEncryptionEnabled" patch: spec: cacheNodeType: cache.micro # invalid value + wait: 60 expect: status: conditions: ACK.Terminal: "True" + ACK.ResourceSynced: + status: "True" + # message: "Expected message ..." - id: "delete_CMD_RG" description: "Delete cluster mode disabled replication group" delete: test$RANDOM_SUFFIX diff --git a/test/e2e/tests/test_scenarios.py b/test/e2e/tests/test_scenarios.py index b2b4fa06..bbaf37c4 100644 --- a/test/e2e/tests/test_scenarios.py +++ b/test/e2e/tests/test_scenarios.py @@ -18,28 +18,25 @@ To add test: add scenario yaml to scenarios/ directory. """ -from declarative_test_fwk import helper, loader, runner +from e2e.declarative_test_fwk import runner, loader, helper import pytest import boto3 import logging -from e2e import service_bootstrap, service_cleanup, service_marker, scenarios_directory, CRD_VERSION, CRD_GROUP, SERVICE_NAME -from acktest.k8s import resource as k8s +from e2e import service_marker, scenarios_directory, resource_directory, CRD_VERSION, CRD_GROUP, SERVICE_NAME +from e2e.bootstrap_resources import get_bootstrap_resources -@helper.resource_helper("ReplicationGroup") -class ReplicationGroupHelper(helper.ResourceHelper): - def assert_expectations(self, verb: str, input_data: dict, expectations: dict, - reference: k8s.CustomResourceReference): - # default assertions - super().assert_expectations(verb, input_data, expectations, reference) +from acktest.k8s import resource as k8s - # perform custom server side checks based on: - # verb, input data to verb, expectations for given resource +@helper.register_resource_helper(resource_kind="ReplicationGroup", resource_plural="ReplicationGroups") +class ReplicationGroupHelper(helper.ResourceHelper): """ - Helper for replication group scenarios + Helper for replication group scenarios. + Overrides methods as required for custom resources. """ + def wait_for_delete(self, reference: k8s.CustomResourceReference): logging.debug(f"ReplicationGroupHelper - wait_for_delete()") ec = boto3.client("elasticache") @@ -51,41 +48,26 @@ def wait_for_delete(self, reference: k8s.CustomResourceReference): @pytest.fixture(scope="session") def input_replacements(): """ - Session scoped fixture to bootstrap service resources and teardown provides input replacements for test scenarios. - Eliminates the need for: - - bootstrap.yaml and - - call to /service_bootstrap.py from test-infra - - call to /service_cleanup.py from test-infra """ - - resources_dict = service_bootstrap.service_bootstrap() + resource_replacements = get_bootstrap_resources().replacement_dict() replacements = { "CRD_VERSION": CRD_VERSION, "CRD_GROUP": CRD_GROUP, - "SERVICE_NAME": SERVICE_NAME, - "SNS_TOPIC_ARN": resources_dict.get("SnsTopicARN"), - "SG_ID": resources_dict.get("SecurityGroupID"), - "USERGROUP_ID": resources_dict.get("UserGroupID"), - "KMS_KEY_ID": resources_dict.get("KmsKeyID"), - "SNAPSHOT_NAME": resources_dict.get("SnapshotName"), - "NON_DEFAULT_USER": resources_dict.get("SnapshotName") + "SERVICE_NAME": SERVICE_NAME } - - yield replacements - # teardown - service_cleanup.service_cleanup(resources_dict) + yield {**resource_replacements, **replacements} @pytest.fixture(params=loader.list_scenarios(scenarios_directory), ids=loader.idfn) def scenario(request, input_replacements): """ - Parameterized fixture - Provides scenarios to execute - Supports parallel execution of scenarios + Parameterized pytest fixture + Provides test scenarios to execute + Supports parallel execution of test scenarios """ scenario_file_path = request.param - scenario = loader.load_scenario(scenario_file_path, input_replacements) + scenario = loader.load_scenario(scenario_file_path, resource_directory, input_replacements) yield scenario runner.teardown(scenario)