diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aae8c2..2a64b51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ [Unreleased]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.10.0...HEAD +### Added + +- Added runtime strategies for rollback as per [chaostoolkit#176][]. + Until now, they were never played + an activity would fail during the hypothesis or if the execution + was interrupted from a control. With the strategies, you can now decide + that they are always applied, never or only when the experiment deviated. + This is a flag passed to the settings as follows: + + ``` + runtime: + rollbacks: + strategy: "always|never|default|deviated" + ``` + + The `"default"` strategy remains backward compatible. + + [chaostoolkit#176]: https://github.com/chaostoolkit/chaostoolkit/issues/176 + ## [1.10.0][] - 2020-06-19 [1.10.0]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.9.0...1.10.0 diff --git a/chaoslib/experiment.py b/chaoslib/experiment.py index 7318615..395f2d6 100644 --- a/chaoslib/experiment.py +++ b/chaoslib/experiment.py @@ -190,6 +190,8 @@ def run_experiment(experiment: Experiment, initialize_global_controls(experiment, config, secrets, settings) initialize_controls(experiment, config, secrets) activity_pool, rollback_pool = get_background_pools(experiment) + rollback_strategy = settings.get("runtime", {}).get( + "rollbacks", {}).get("strategy", "default") experiment["title"] = substitute(experiment["title"], config, secrets) logger.info("Running experiment: {t}".format(t=experiment["title"])) @@ -254,6 +256,34 @@ def run_experiment(experiment: Experiment, "leaving without applying rollbacks.") else: journal["status"] = journal["status"] or "completed" + + has_deviated = journal["deviated"] + journal_status = journal["status"] + play_rollbacks = False + if rollback_strategy == "always": + logger.warning( + "Rollbacks were explicitly requested to be played") + play_rollbacks = True + elif rollback_strategy == "never": + logger.warning( + "Rollbacks were explicitly requested to not be played") + play_rollbacks = False + elif rollback_strategy == "default" and \ + journal_status not in ["failed", "interrupted"]: + play_rollbacks = True + elif rollback_strategy == "deviated": + if has_deviated: + logger.warning( + "Rollbacks will be played only because the experiment " + "deviated") + play_rollbacks = True + else: + logger.warning( + "Rollbacks werre explicitely requested to be played only " + "if the experiment deviated. Since this is not the case, " + "we will not play them.") + + if play_rollbacks: try: journal["rollbacks"] = apply_rollbacks( experiment, config, secrets, rollback_pool, dry) @@ -269,8 +299,7 @@ def run_experiment(experiment: Experiment, journal["end"] = datetime.utcnow().isoformat() journal["duration"] = time.time() - started_at - has_deviated = journal["deviated"] - status = "deviated" if has_deviated else journal["status"] + status = "deviated" if has_deviated else journal_status logger.info( "Experiment ended with status: {s}".format(s=status)) diff --git a/tests/fixtures/actions.py b/tests/fixtures/actions.py index 9f6b945..a069ed3 100644 --- a/tests/fixtures/actions.py +++ b/tests/fixtures/actions.py @@ -1,5 +1,48 @@ -# -*- coding: utf-8 -*- -import sys +EmptyAction = {} -EmptyAction = {} +DoNothingAction = { + "name": "a name", + "type": "action", + "provider": { + "type": "python", + "module": "fixtures.fakeext", + "func": "do_nothing" + } +} + + +EchoAction = { + "name": "a name", + "type": "action", + "provider": { + "type": "python", + "module": "fixtures.fakeext", + "func": "echo_message", + "arguments": { + "message": "kaboom" + } + } +} + + +FailAction = { + "name": "a name", + "type": "action", + "provider": { + "type": "python", + "module": "fixtures.fakeext", + "func": "force_failed_activity" + } +} + + +InterruptAction = { + "name": "a name", + "type": "action", + "provider": { + "type": "python", + "module": "fixtures.fakeext", + "func": "force_interrupting_experiment" + } +} diff --git a/tests/fixtures/experiments.py b/tests/fixtures/experiments.py index af8c0cf..26d2387 100644 --- a/tests/fixtures/experiments.py +++ b/tests/fixtures/experiments.py @@ -2,6 +2,8 @@ from copy import deepcopy import os +from fixtures.actions import DoNothingAction, EchoAction, FailAction, \ + InterruptAction from fixtures.probes import BackgroundPythonModuleProbe, MissingFuncArgProbe, \ PythonModuleProbe, PythonModuleProbeWithBoolTolerance, \ PythonModuleProbeWithExternalTolerance, PythonModuleProbeWithLongPause, \ @@ -11,7 +13,7 @@ PythonModuleProbeWithProcessStatusTolerance, \ PythonModuleProbeWithProcessFailedStatusTolerance, \ PythonModuleProbeWithProcesStdoutTolerance, \ - PythonModuleProbeWithHTTPStatusToleranceDeviation + PythonModuleProbeWithHTTPStatusToleranceDeviation, FailProbe Secrets = {} @@ -421,3 +423,75 @@ path: {} timeout: 30 """.format(os.path.abspath(__file__)) + + +ExperimentWithRegularRollback = { + "title": "do cats live in the Internet?", + "description": "an experiment of importance", + "steady-state-hypothesis": { + "title": "hello" + }, + "method": [ + EchoAction + ], + "rollbacks": [ + EchoAction + ] +} + + +ExperimentWithFailedActionInMethodAndARollback = { + "title": "do cats live in the Internet?", + "description": "an experiment of importance", + "steady-state-hypothesis": { + "title": "hello" + }, + "method": [ + FailAction + ], + "rollbacks": [ + EchoAction + ] +} + + +ExperimentWithFailedActionInSSHAndARollback = { + "title": "do cats live in the Internet?", + "description": "an experiment of importance", + "steady-state-hypothesis": { + "title": "hello", + "probes": [ + FailProbe + ] + }, + "method": [ + DoNothingAction + ], + "rollbacks": [ + EchoAction + ] +} + + +ExperimentWithInterruptedExperimentAndARollback = { + "title": "do cats live in the Internet?", + "description": "an experiment of importance", + "steady-state-hypothesis": { + "title": "hello" + }, + "method": [ + deepcopy(EchoAction) + ], + "rollbacks": [ + EchoAction + ] +} +ExperimentWithInterruptedExperimentAndARollback["method"][0]["controls"] = [ + { + "name": "dummy", + "provider": { + "type": "python", + "module": "fixtures.interruptexperiment" + } + } +] diff --git a/tests/fixtures/fakeext.py b/tests/fixtures/fakeext.py index 1d5dba2..c771813 100644 --- a/tests/fixtures/fakeext.py +++ b/tests/fixtures/fakeext.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from typing import List -from chaoslib.exceptions import InterruptExecution +from chaoslib.exceptions import ActivityFailed, InterruptExecution __all__ = ["many_args", "many_args_with_rich_types", "no_args_docstring", "no_args", "one_arg", "one_untyped_arg", "one_arg_with_default", @@ -66,3 +66,20 @@ def many_args_with_rich_types(message: str, recipients: List[str], Many arguments with rich types. """ pass + + +def do_nothing(): + pass + + +def echo_message(message: str) -> str: + print(message) + return message + + +def force_failed_activity(): + raise ActivityFailed() + + +def force_interrupting_experiment(): + raise InterruptExecution() diff --git a/tests/fixtures/interruptexperiment.py b/tests/fixtures/interruptexperiment.py new file mode 100644 index 0000000..a901c06 --- /dev/null +++ b/tests/fixtures/interruptexperiment.py @@ -0,0 +1,5 @@ +from chaoslib.exceptions import InterruptExecution + + +def after_activity_control(**kwargs): + raise InterruptExecution() diff --git a/tests/fixtures/probes.py b/tests/fixtures/probes.py index 6e5188e..33204a3 100644 --- a/tests/fixtures/probes.py +++ b/tests/fixtures/probes.py @@ -373,3 +373,14 @@ def must_be_in_range(a: int, b: int, value: Any = None) -> bool: raise ActivityFailed("body is not in expected range") else: return True + +FailProbe = { + "name": "a name", + "type": "probe", + "tolerance": True, + "provider": { + "type": "python", + "module": "fixtures.fakeext", + "func": "force_failed_activity" + } +} diff --git a/tests/test_experiment.py b/tests/test_experiment.py index e2cf689..18d2648 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -311,3 +311,131 @@ def test_dry_run_should_not_pause_before(): pause_before_duration = int(experiment["method"][1]["pauses"]["before"]) assert experiment_run_time < pause_before_duration + + +def test_rollback_default_strategy_does_not_run_on_failed_activity_in_ssh(): + experiment = experiments.ExperimentWithFailedActionInSSHAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "default" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "failed" + assert len(journal["rollbacks"]) == 0 + + +def test_rollback_default_strategy_runs_on_failed_activity_in_method(): + experiment = experiments.ExperimentWithFailedActionInMethodAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "default" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "completed" + assert len(journal["rollbacks"]) == 1 + + +def test_rollback_default_strategy_does_not_run_on_interrupted_experiment_in_method(): + experiment = experiments.ExperimentWithInterruptedExperimentAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "always" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "interrupted" + assert len(journal["rollbacks"]) == 1 + + +def test_rollback_always_strategy_runs_on_failed_activity_in_ssh(): + experiment = experiments.ExperimentWithFailedActionInSSHAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "always" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "failed" + assert len(journal["rollbacks"]) == 1 + + +def test_rollback_always_strategy_runs_on_interrupted_experiment_in_method(): + experiment = experiments.ExperimentWithInterruptedExperimentAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "always" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "interrupted" + assert len(journal["rollbacks"]) == 1 + + +def test_rollback_always_strategy_runs_on_failed_activity_in_method(): + experiment = experiments.ExperimentWithFailedActionInMethodAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "always" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "completed" + assert len(journal["rollbacks"]) == 1 + + +def test_rollback_never_strategy_does_not_run_on_failed_activity_in_ssh(): + experiment = experiments.ExperimentWithFailedActionInSSHAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "never" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "failed" + assert len(journal["rollbacks"]) == 0 + + +def test_rollback_never_strategy_does_not_run_on_interrupted_experiment_in_method(): + experiment = experiments.ExperimentWithInterruptedExperimentAndARollback + #experiment["dry"] = True + settings = { + "runtime": { + "rollbacks": { + "strategy": "never" + } + } + } + + journal = run_experiment(experiment, settings) + assert journal["status"] == "interrupted" + assert len(journal["rollbacks"]) == 0