From 76d8803a5dd0940054b5a22b16cbe43b6c677c9e Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 17 Dec 2018 12:17:47 -0500 Subject: [PATCH] Add a --fail_fast switch to common test This switch aborts the rebar3 run after the first test failure is reported. Since Common Test does not support this feature and it can be useful for interactive development (a demand from one of the teams I work with), I decided to add the feature as an experimental CT hook. The hook notices any test failure, and forcefully aborts the current script execution, but only during the initialization phase of the next test; this ensure that all other hooks for a post-test failure have the time to do things right. The CT logs _will_ be interupted, and hooks of all kinds may also suffer. Since this might be a bit tricky to support internally, I will be fine with a review that results in "no, Fred. Please test this in a project plugin first" or if I get told to use profiles to mature the feature (i.e. `rebar3 as failfast ct ...`) with a custom hook declaration. --- src/cth_fail_fast.erl | 118 ++++++++++++++++++++++++++++++++++ src/rebar_prv_common_test.erl | 26 ++++++-- 2 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 src/cth_fail_fast.erl diff --git a/src/cth_fail_fast.erl b/src/cth_fail_fast.erl new file mode 100644 index 000000000..13b35570d --- /dev/null +++ b/src/cth_fail_fast.erl @@ -0,0 +1,118 @@ +-module(cth_fail_fast). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3, on_tc_skip/4]). + +-export([terminate/1]). + +%% We work by setting an 'abort' variable on each test case that fails +%% and then triggering the failure before starting the next test. This +%% ensures that all other hooks have run for the same event, and +%% simplifies error reporting. +-record(state, {abort=false}). + +%% @doc Return a unique id for this CTH. +id(_Opts) -> + {?MODULE, make_ref()}. + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(_Id, _Opts) -> + {ok, #state{}}. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(_Suite,_Config,#state{abort=true}) -> + abort(); +pre_init_per_suite(_Suite,Config,State) -> + {Config, State}. + +%% @doc Called after init_per_suite. +post_init_per_suite(_Suite,_Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(_Suite,_Config,#state{abort=true}) -> + abort(); +pre_end_per_suite(_Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(_Suite,_Config,Return,State) -> + {Return, State}. + +%% @doc Called before each init_per_group. +pre_init_per_group(_Group,_Config,#state{abort=true}) -> + abort(); +pre_init_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(_Group,_Config,Return, State) -> + {Return, State}. + +%% @doc Called after each end_per_group. +pre_end_per_group(_Group,_Config,#state{abort=true}) -> + abort(); +pre_end_per_group(_Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(_Group,_Config,Return, State) -> + {Return, State}. + +%% @doc Called before each test case. +pre_init_per_testcase(_TC,_Config,#state{abort=true}) -> + abort(); +pre_init_per_testcase(_TC,Config,State) -> + {Config, State}. + +%% @doc Called after each test case. +post_end_per_testcase(_TC,_Config,ok,State) -> + {ok, State}; +post_end_per_testcase(_TC,_Config,Error,State) -> + {Error, State#state{abort=true}}. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(_TC, _Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (>= 19.3) +on_tc_skip(_Suite, _TC, {tc_auto_skip, _}, State) -> + State#state{abort=true}; +on_tc_skip(_Suite, _TC, _Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. (Pre-19.3) +on_tc_skip(_TC, {tc_auto_skip, _}, State) -> + State#state{abort=true}; +on_tc_skip(_TC, _Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(#state{}) -> + ok. + +%%% Helpers +abort() -> + io:format(user, "Detected test failure. Aborting~n", []), + halt(1). diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl index 3d3bd8a8d..c31c060e7 100644 --- a/src/rebar_prv_common_test.erl +++ b/src/rebar_prv_common_test.erl @@ -171,6 +171,9 @@ transform_opts([{cover, _}|Rest], Acc) -> %% drop verbose from opts, ct doesn't care about it transform_opts([{verbose, _}|Rest], Acc) -> transform_opts(Rest, Acc); +%% drop fail_fast from opts, ct doesn't care about it +transform_opts([{fail_fast, _}|Rest], Acc) -> + transform_opts(Rest, Acc); %% getopt should handle anything else transform_opts([Opt|Rest], Acc) -> transform_opts(Rest, [Opt|Acc]). @@ -224,15 +227,21 @@ ensure_opts([V|Rest], Acc) -> ensure_opts(Rest, [V|Acc]). add_hooks(Opts, State) -> + FailFast = case fails_fast(State) of + true -> [cth_fail_fast]; + false -> [] + end, case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of {false, _} -> Opts; {Other, false} -> - [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts]; + [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), + cth_retry] ++ FailFast} | Opts]; {Other, {ct_hooks, Hooks}} -> %% Make sure hooks are there once only. - ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry], - AllReadableHooks = [cth_readable_failonly, cth_retry, + ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), + cth_retry] ++ FailFast, + AllReadableHooks = [cth_readable_failonly, cth_retry, cth_fail_fast, cth_readable_shell, cth_readable_compact_shell], NewHooks = (Hooks -- AllReadableHooks) ++ ReadableHooks, lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) @@ -445,6 +454,10 @@ readable(State) -> undefined -> rebar_state:get(State, ct_readable, compact) end. +fails_fast(State) -> + {RawOpts, _} = rebar_state:command_parsed_args(State), + proplists:get_value(fail_fast, RawOpts) == true. + test_dirs(State, Apps, Opts) -> case proplists:get_value(spec, Opts) of undefined -> @@ -773,7 +786,8 @@ ct_opts(_State) -> {setcookie, undefined, "setcookie", atom, help(setcookie)}, {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list {compile_only, undefined, "compile_only", boolean, help(compile_only)}, - {retry, undefined, "retry", boolean, help(retry)} + {retry, undefined, "retry", boolean, help(retry)}, + {fail_fast, undefined, "fail_fast", {boolean, false}, help(fail_fast)} ]. help(compile_only) -> @@ -846,5 +860,9 @@ help(setcookie) -> "Sets the cookie if the node is distributed"; help(retry) -> "Experimental feature. If any specification for previously failing test is found, runs them."; +help(fail_fast) -> + "Experimental feature. If any test fails, the run is aborted. Since common test does not " + "support this natively, we abort the rebar3 run on a failure. This May break CT's disk logging and " + "other rebar3 features."; help(_) -> "".