diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d9c1ddf..fce1d506 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,8 @@ v3.0.2 *Release date: In development* - `context.storage` must be initialized before dynamic environment steps in `before_feature` method +- Mark scenario as failed when a dynamic environment step fails in `before_scenario` +- Mark all feature scenarios as failed when a dynamic environment step fails in `before_feature` and `after_feature` v3.0.1 ------ diff --git a/requirements_dev.txt b/requirements_dev.txt index 91d3b593..ae10c99c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -9,3 +9,4 @@ flake8~=6.0; python_version >= '3.8' build~=0.10 wheel~=0.40 twine~=4.0 +behave==1.2.6 # behave tests diff --git a/toolium/behave/env_utils.py b/toolium/behave/env_utils.py index e73cd786..a7b3d77b 100644 --- a/toolium/behave/env_utils.py +++ b/toolium/behave/env_utils.py @@ -18,15 +18,14 @@ import sys import warnings -from pkg_resources import parse_version -# constants -# pre-actions in feature files +# Actions types defined in feature files ACTIONS_BEFORE_FEATURE = 'actions before the feature' ACTIONS_BEFORE_SCENARIO = 'actions before each scenario' ACTIONS_AFTER_SCENARIO = 'actions after each scenario' ACTIONS_AFTER_FEATURE = 'actions after the feature' -KEYWORDS = ['Setup', 'Check', 'Given', 'When', 'Then', 'And', 'But'] # prefix in steps to actions +# Valid prefix in action steps +KEYWORDS = ['Setup', 'Check', 'Given', 'When', 'Then', 'And', 'But'] GIVEN_PREFIX = 'Given' TABLE_SEPARATOR = '|' STEP_TEXT_SEPARATORS = ['"""', "'''"] @@ -118,6 +117,19 @@ def after_scenario(context, scenario): def after_feature(context, feature): # ---- actions after feature ---- context.dyn_env.execute_after_feature_steps(context) + + Error management with behave and dynamic environment: + * Error in before_feature: + before_scenario and after_scenario are not executed + after_feature is executed + each scenario is marked as failed + * Error in before_scenario: + after_scenario is executed + the scenario is marked as failed + * Error in after_scenario: + the scenario status is not changed + * Error in after_feature: + the scenarios status are not changed """ def __init__(self, **kwargs): @@ -134,6 +146,7 @@ def __init__(self, **kwargs): self.scenario_counter = 0 self.feature_error = False self.scenario_error = False + self.before_error_message = None def init_actions(self): """clear actions lists""" @@ -212,29 +225,20 @@ def __execute_steps_by_action(self, context, action): self.logger.by_console('\n') for item in self.actions[action]: - self.scenario_error = False try: self.__print_step_by_console(item) + self.logger.debug('Executing step defined in %s: %s' % (action, repr(item))) context.execute_steps('''%s%s''' % (GIVEN_PREFIX, self.__remove_prefix(item))) - self.logger.debug('step defined in pre-actions: %s' % repr(item)) except Exception as exc: - self.feature_error = action in [ACTIONS_BEFORE_FEATURE] - self.scenario_error = action in [ACTIONS_BEFORE_SCENARIO] + if action == ACTIONS_BEFORE_FEATURE: + self.feature_error = True + self.before_error_message = exc + elif action == ACTIONS_BEFORE_SCENARIO: + self.scenario_error = True + self.before_error_message = exc self.logger.error(exc) - self.error_exception = exc break - def reset_error_status(self): - """ - Check if the dyn_env has got any exception when executing the steps and restore the value of status to False. - :return: True if any exception has been raised when executing steps - """ - try: - return self.feature_error or self.scenario_error - finally: - self.feature_error = False - self.scenario_error = False - def execute_before_feature_steps(self, context): """ actions before the feature @@ -243,8 +247,9 @@ def execute_before_feature_steps(self, context): """ self.__execute_steps_by_action(context, ACTIONS_BEFORE_FEATURE) - if context.dyn_env.feature_error: - # Mark this Feature as skipped. Steps will not be executed. + if self.feature_error: + # Mark this Feature as skipped to do not execute any Scenario + # Status will be changed to failed in after_feature method context.feature.mark_skipped() def execute_before_scenario_steps(self, context): @@ -253,11 +258,11 @@ def execute_before_scenario_steps(self, context): :param context: It’s a clever place where you and behave can store information to share around, automatically managed by behave. """ - if not self.feature_error: - self.__execute_steps_by_action(context, ACTIONS_BEFORE_SCENARIO) + self.__execute_steps_by_action(context, ACTIONS_BEFORE_SCENARIO) - if context.dyn_env.scenario_error: - # Mark this Scenario as skipped. Steps will not be executed. + if self.scenario_error: + # Mark this Scenario as skipped to do not execute any step + # Status will be changed to failed in after_scenario method context.scenario.mark_skipped() def execute_after_scenario_steps(self, context): @@ -266,13 +271,16 @@ def execute_after_scenario_steps(self, context): :param context: It’s a clever place where you and behave can store information to share around, automatically managed by behave. """ - if not self.feature_error: - self.__execute_steps_by_action(context, ACTIONS_AFTER_SCENARIO) + self.__execute_steps_by_action(context, ACTIONS_AFTER_SCENARIO) - # Behave dynamic environment: Fail all steps if dyn_env has got any error and reset it - if self.reset_error_status(): + # Mark first step as failed when before_scenario has failed + if self.scenario_error: + error_message = self.before_error_message + self.scenario_error = False + self.before_error_message = None context.scenario.reset() - context.dyn_env.fail_first_step_precondition_exception(context.scenario) + self.fail_first_step_precondition_exception(context.scenario) + raise Exception(f'Before scenario steps have failed: {error_message}') def execute_after_feature_steps(self, context): """ @@ -282,13 +290,16 @@ def execute_after_feature_steps(self, context): """ self.__execute_steps_by_action(context, ACTIONS_AFTER_FEATURE) - # Behave dynamic environment: Fail all steps if dyn_env has got any error and reset it - if self.reset_error_status(): + # Mark first step of each scenario as failed when before_feature has failed + if self.feature_error: + error_message = self.before_error_message + self.feature_error = False + self.before_error_message = None context.feature.reset() for scenario in context.feature.walk_scenarios(): if scenario.should_run(context.config): - context.dyn_env.fail_first_step_precondition_exception(scenario) - raise Exception("Preconditions failed during the execution") + self.fail_first_step_precondition_exception(scenario) + raise Exception(f'Before feature steps have failed: {error_message}') def fail_first_step_precondition_exception(self, scenario): """ @@ -296,17 +307,9 @@ def fail_first_step_precondition_exception(self, scenario): This is needed because xUnit exporter in Behave fails if there are not failed steps. :param scenario: Behave's Scenario """ - - try: - import behave - if parse_version(behave.__version__) < parse_version('1.2.6'): - status = 'failed' - else: - status = behave.model_core.Status.failed - except ImportError as exc: - self.logger.error(exc) - raise - - scenario.steps[0].status = status - scenario.steps[0].exception = Exception("Preconditions failed") - scenario.steps[0].error_message = str(self.error_exception) + if len(scenario.steps) > 0: + # Behave is an optional dependency in toolium, so it is imported here + from behave.model_core import Status + scenario.steps[0].status = Status.failed + scenario.steps[0].exception = Exception('Preconditions failed') + scenario.steps[0].error_message = str(self.before_error_message) diff --git a/toolium/test/behave/test_env_utils.py b/toolium/test/behave/test_env_utils.py new file mode 100644 index 00000000..2ce9c3f8 --- /dev/null +++ b/toolium/test/behave/test_env_utils.py @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2023 Telefónica Investigación y Desarrollo, S.A.U. +This file is part of Toolium. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import mock +import pytest + +from toolium.behave.env_utils import (DynamicEnvironment, ACTIONS_BEFORE_FEATURE, ACTIONS_BEFORE_SCENARIO, + ACTIONS_AFTER_SCENARIO, ACTIONS_AFTER_FEATURE) + + +@pytest.fixture +def context(): + context = mock.MagicMock() + return context + + +@pytest.fixture +def dyn_env(): + dyn_env = DynamicEnvironment() + dyn_env.actions[ACTIONS_BEFORE_FEATURE] = ['Given before feature step'] + dyn_env.actions[ACTIONS_BEFORE_SCENARIO] = ['Given before scenario step'] + dyn_env.actions[ACTIONS_AFTER_SCENARIO] = ['Then after scenario step'] + dyn_env.actions[ACTIONS_AFTER_FEATURE] = ['Then after feature step'] + return dyn_env + + +# TODO: add tests for get_steps_from_feature_description method + + +def test_execute_before_feature_steps_without_actions(context): + # Create dynamic environment without actions + dyn_env = DynamicEnvironment() + + dyn_env.execute_before_feature_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_not_called() + context.feature.mark_skipped.assert_not_called() + + +def test_execute_before_feature_steps_passed_actions(context, dyn_env): + dyn_env.execute_before_feature_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_called_with('Given before feature step') + context.feature.mark_skipped.assert_not_called() + + +def test_execute_before_feature_steps_failed_actions(context, dyn_env): + context.execute_steps.side_effect = Exception('Exception in before feature step') + + dyn_env.execute_before_feature_steps(context) + assert dyn_env.feature_error is True + assert dyn_env.scenario_error is False + context.execute_steps.assert_called_with('Given before feature step') + context.feature.mark_skipped.assert_called_once_with() + + +def test_execute_before_scenario_steps_without_actions(context): + # Create dynamic environment without actions + dyn_env = DynamicEnvironment() + + dyn_env.execute_before_scenario_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_not_called() + context.scenario.mark_skipped.assert_not_called() + + +def test_execute_before_scenario_steps_passed_actions(context, dyn_env): + dyn_env.execute_before_scenario_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_called_with('Given before scenario step') + context.scenario.mark_skipped.assert_not_called() + + +def test_execute_before_scenario_steps_failed_actions(context, dyn_env): + context.execute_steps.side_effect = Exception('Exception in before scenario step') + + dyn_env.execute_before_scenario_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is True + context.execute_steps.assert_called_with('Given before scenario step') + context.scenario.mark_skipped.assert_called_once_with() + + +def test_execute_after_scenario_steps_without_actions(context): + # Create dynamic environment without actions + dyn_env = DynamicEnvironment() + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + dyn_env.execute_after_scenario_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_not_called() + context.scenario.reset.assert_not_called() + dyn_env.fail_first_step_precondition_exception.assert_not_called() + + +def test_execute_after_scenario_steps_passed_actions(context, dyn_env): + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + dyn_env.execute_after_scenario_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_called_with('Given after scenario step') + context.scenario.reset.assert_not_called() + dyn_env.fail_first_step_precondition_exception.assert_not_called() + + +def test_execute_after_scenario_steps_failed_actions(context, dyn_env): + context.execute_steps.side_effect = Exception('Exception in after scenario step') + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + dyn_env.execute_after_scenario_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_called_with('Given after scenario step') + context.scenario.reset.assert_not_called() + dyn_env.fail_first_step_precondition_exception.assert_not_called() + + +def test_execute_after_scenario_steps_failed_before_scenario(context, dyn_env): + # Before scenario step fails + dyn_env.scenario_error = True + dyn_env.before_error_message = 'Exception in before scenario step' + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + with pytest.raises(Exception) as excinfo: + dyn_env.execute_after_scenario_steps(context) + assert 'Before scenario steps have failed: Exception in before scenario step' == str(excinfo.value) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + assert dyn_env.before_error_message is None + context.execute_steps.assert_called_with('Given after scenario step') + context.scenario.reset.assert_called() + dyn_env.fail_first_step_precondition_exception.assert_called() + + +def test_execute_after_scenario_steps_failed_actions_failed_before_scenario(context, dyn_env): + # Before scenario step fails + dyn_env.scenario_error = True + dyn_env.before_error_message = 'Exception in before scenario step' + context.execute_steps.side_effect = Exception('Exception in after scenario step') + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + with pytest.raises(Exception) as excinfo: + dyn_env.execute_after_scenario_steps(context) + assert 'Before scenario steps have failed: Exception in before scenario step' == str(excinfo.value) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + assert dyn_env.before_error_message is None + context.execute_steps.assert_called_with('Given after scenario step') + context.scenario.reset.assert_called() + dyn_env.fail_first_step_precondition_exception.assert_called() + + +def test_execute_after_feature_steps_without_actions(context): + # Create dynamic environment without actions + dyn_env = DynamicEnvironment() + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + dyn_env.execute_after_feature_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_not_called() + context.feature.reset.assert_not_called() + dyn_env.fail_first_step_precondition_exception.assert_not_called() + + +def test_execute_after_feature_steps_passed_actions(context, dyn_env): + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + dyn_env.execute_after_feature_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_called_with('Given after feature step') + context.feature.reset.assert_not_called() + dyn_env.fail_first_step_precondition_exception.assert_not_called() + + +def test_execute_after_feature_steps_failed_actions(context, dyn_env): + context.execute_steps.side_effect = Exception('Exception in before feature step') + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + + dyn_env.execute_after_feature_steps(context) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + context.execute_steps.assert_called_with('Given after feature step') + context.feature.reset.assert_not_called() + dyn_env.fail_first_step_precondition_exception.assert_not_called() + + +def test_execute_after_feature_steps_failed_before_feature(context, dyn_env): + # Before feature step fails + dyn_env.feature_error = True + dyn_env.before_error_message = 'Exception in before feature step' + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + context.feature.walk_scenarios.return_value = [context.scenario] + + with pytest.raises(Exception) as excinfo: + dyn_env.execute_after_feature_steps(context) + assert 'Before feature steps have failed: Exception in before feature step' == str(excinfo.value) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + assert dyn_env.before_error_message is None + context.execute_steps.assert_called_with('Given after feature step') + context.feature.reset.assert_called_once_with() + dyn_env.fail_first_step_precondition_exception.assert_called_once_with(context.scenario) + + +def test_execute_after_feature_steps_failed_actions_failed_before_feature(context, dyn_env): + # Before feature step fails + dyn_env.feature_error = True + dyn_env.before_error_message = 'Exception in before feature step' + dyn_env.fail_first_step_precondition_exception = mock.MagicMock() + context.execute_steps.side_effect = Exception('Exception in after feature step') + context.feature.walk_scenarios.return_value = [context.scenario] + + with pytest.raises(Exception) as excinfo: + dyn_env.execute_after_feature_steps(context) + assert 'Before feature steps have failed: Exception in before feature step' == str(excinfo.value) + assert dyn_env.feature_error is False + assert dyn_env.scenario_error is False + assert dyn_env.before_error_message is None + context.execute_steps.assert_called_with('Given after feature step') + context.feature.reset.assert_called_once_with() + dyn_env.fail_first_step_precondition_exception.assert_called_once_with(context.scenario) + + +def test_fail_first_step_precondition_exception(dyn_env): + scenario = mock.MagicMock() + step1 = mock.MagicMock() + step2 = mock.MagicMock() + scenario.steps = [step1, step2] + dyn_env.before_error_message = 'Exception in before feature step' + + dyn_env.fail_first_step_precondition_exception(scenario) + assert step1.status == 'failed' + assert str(step1.exception) == 'Preconditions failed' + assert step1.error_message == 'Exception in before feature step' + + +def test_fail_first_step_precondition_exception_without_steps(dyn_env): + scenario = mock.MagicMock() + scenario.steps = [] + dyn_env.before_error_message = 'Exception in before feature step' + + dyn_env.fail_first_step_precondition_exception(scenario) diff --git a/toolium/test/behave/test_environment.py b/toolium/test/behave/test_environment.py index 2cd12cc5..e05028fd 100644 --- a/toolium/test/behave/test_environment.py +++ b/toolium/test/behave/test_environment.py @@ -274,10 +274,9 @@ def test_after_feature_with_failed_preconditions(DriverWrappersPool): context.dyn_env = mock.MagicMock() context.dyn_env.execute_after_feature_steps.side_effect = Exception('Preconditions failed') - try: + with pytest.raises(Exception) as excinfo: after_feature(context, feature) - except Exception: - pass + assert 'Preconditions failed' == str(excinfo.value) # Check that close_drivers is called DriverWrappersPool.close_drivers.assert_called_once_with(scope='module', test_name='name', test_passed=True)