Skip to content

Commit

Permalink
Make {repeat*,N} property in group execute the group N times exactly
Browse files Browse the repository at this point in the history 
To be consistent with the behaviour of the run_test repeat flag/option, the repeat* group property has been changed to specify absolute number of test runs. Previously {repeat,N} meant "execute the group 1 time + N repeats". Now it means "execute the group N times".
  • Loading branch information
@RaimoNiskanen
Peter Andersson authored and RaimoNiskanen committed Jun 9, 2010
1 parent 2dbeff6 commit 10a06d2
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 56 deletions.
40 changes: 20 additions & 20 deletions lib/common_test/test/ct_groups_test_1_SUITE.erl
View file
Expand Up @@ -335,14 +335,14 @@ test_events(groups_suite_2) ->
{?eh,tc_start,{groups_12_SUITE,testcase_2a}},
{?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}},

[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}},
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3a}},
{?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3b}},
{?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}},
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}],
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}],

[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}},
Expand Down Expand Up @@ -529,14 +529,14 @@ test_events(groups_suites_1) ->
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_2a}},
{?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}},
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}},
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3a}},
{?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3b}},
{?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}},
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}],
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}],
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3a}},
Expand Down Expand Up @@ -715,14 +715,14 @@ test_events(groups_dir_1) ->
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_2a}},
{?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}},
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}},
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3a}},
{?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3b}},
{?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}},
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}],
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}],
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3a}},
Expand Down Expand Up @@ -902,14 +902,14 @@ test_events(groups_dirs_1) ->
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_2a}},
{?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}},
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}},
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3a}},
{?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3b}},
{?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}},
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}],
{?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}],
[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}},
{?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}},
{?eh,tc_start,{groups_12_SUITE,testcase_3a}},
Expand Down Expand Up @@ -1130,17 +1130,17 @@ test_events(groups_dirs_1) ->
{?eh,tc_start,{groups_22_SUITE,testcase_2a}},
{?eh,tc_done,{groups_22_SUITE,testcase_2a,ok}},
[{?eh,tc_start,
{groups_22_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}},
{groups_22_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,
{groups_22_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}},
{groups_22_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}},
{?eh,tc_start,{groups_22_SUITE,testcase_3a}},
{?eh,tc_done,{groups_22_SUITE,testcase_3a,ok}},
{?eh,tc_start,{groups_22_SUITE,testcase_3b}},
{?eh,tc_done,{groups_22_SUITE,testcase_3b,ok}},
{?eh,tc_start,
{groups_22_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}},
{groups_22_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}},
{?eh,tc_done,
{groups_22_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}],
{groups_22_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}],
[{?eh,tc_start,
{groups_22_SUITE,{init_per_group,test_group_3,[]}}},
{?eh,tc_done,
Expand Down
6 changes: 3 additions & 3 deletions lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_12_SUITE.erl
View file
Expand Up @@ -37,7 +37,7 @@ groups() ->

{test_group_2, [parallel], [testcase_2a,

{test_group_3, [{repeat,1}],
{test_group_3, [{repeat,2}],
[testcase_3a, testcase_3b]},

testcase_2b]},
Expand Down Expand Up @@ -102,8 +102,8 @@ init_per_group(Group, Config) ->
io_lib:format("shuffled, ~w", [S]);
{test_group_1b,[{name,test_group_1b},parallel]} -> "parallel";
{test_group_2,[{name,test_group_2},parallel]} -> "parallel";
{test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1";
{test_group_3,[{name,test_group_3}]} -> "repeat 0";
{test_group_3,[{name,test_group_3},{repeat,2}]} -> "repeat 2";
{test_group_3,[{name,test_group_3}]} -> "repeat 1";
{test_group_4,[{name,test_group_4}]} -> ok;
{test_group_5,[{name,test_group_5},parallel]} -> "parallel";
{test_group_6,[{name,test_group_6},parallel]} -> "parallel";
Expand Down
6 changes: 3 additions & 3 deletions lib/common_test/test/ct_groups_test_1_SUITE_data/groups_2/test/groups_22_SUITE.erl
View file
Expand Up @@ -37,7 +37,7 @@ groups() ->

{test_group_2, [parallel], [testcase_2a,

{test_group_3, [{repeat,1}],
{test_group_3, [{repeat,2}],
[testcase_3a, testcase_3b]},

testcase_2b]},
Expand Down Expand Up @@ -102,8 +102,8 @@ init_per_group(Group, Config) ->
io_lib:format("shuffled, ~w", [S]);
{test_group_1b,[{name,test_group_1b},parallel]} -> "parallel";
{test_group_2,[{name,test_group_2},parallel]} -> "parallel";
{test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1";
{test_group_3,[{name,test_group_3}]} -> "repeat 0";
{test_group_3,[{name,test_group_3},{repeat,2}]} -> "repeat 2";
{test_group_3,[{name,test_group_3}]} -> "repeat 1";
{test_group_4,[{name,test_group_4}]} -> ok;
{test_group_5,[{name,test_group_5},parallel]} -> "parallel";
{test_group_6,[{name,test_group_6},parallel]} -> "parallel";
Expand Down
32 changes: 16 additions & 16 deletions lib/common_test/test/ct_groups_test_2_SUITE.erl
View file
Expand Up @@ -184,19 +184,19 @@ test_events(repeat_1) ->
{?eh,tc_start,{repeat_1_SUITE,init_per_suite}},
{?eh,tc_done,{repeat_1_SUITE,init_per_suite,ok}},
[{?eh,tc_start,
{repeat_1_SUITE,{init_per_group,test_group_1,[{repeat,1}]}}},
{repeat_1_SUITE,{init_per_group,test_group_1,[{repeat,2}]}}},
{?eh,tc_done,
{repeat_1_SUITE,{init_per_group,test_group_1,[{repeat,1}]},ok}},
{repeat_1_SUITE,{init_per_group,test_group_1,[{repeat,2}]},ok}},
{?eh,tc_start,{repeat_1_SUITE,testcase_1a}},
{?eh,tc_done,{repeat_1_SUITE,testcase_1a,ok}},
{?eh,test_stats,{1,0,{0,0}}},
{?eh,tc_start,{repeat_1_SUITE,testcase_1b}},
{?eh,tc_done,{repeat_1_SUITE,testcase_1b,ok}},
{?eh,test_stats,{2,0,{0,0}}},
{?eh,tc_start,
{repeat_1_SUITE,{end_per_group,test_group_1,[{repeat,1}]}}},
{repeat_1_SUITE,{end_per_group,test_group_1,[{repeat,2}]}}},
{?eh,tc_done,
{repeat_1_SUITE,{end_per_group,test_group_1,[{repeat,1}]},ok}}],
{repeat_1_SUITE,{end_per_group,test_group_1,[{repeat,2}]},ok}}],
[{?eh,tc_start,
{repeat_1_SUITE,{init_per_group,test_group_1,[]}}},
{?eh,tc_done,
Expand All @@ -212,35 +212,35 @@ test_events(repeat_1) ->
{?eh,tc_done,
{repeat_1_SUITE,{end_per_group,test_group_1,[]},ok}}],
[{?eh,tc_start,
{repeat_1_SUITE,{init_per_group,test_group_2,[{repeat,0}]}}},
{repeat_1_SUITE,{init_per_group,test_group_2,[]}}},
{?eh,tc_done,
{repeat_1_SUITE,{init_per_group,test_group_2,[{repeat,0}]},ok}},
{repeat_1_SUITE,{init_per_group,test_group_2,[]},ok}},
{?eh,tc_start,{repeat_1_SUITE,testcase_2a}},
{?eh,tc_done,{repeat_1_SUITE,testcase_2a,ok}},
{?eh,test_stats,{5,0,{0,0}}},
{?eh,tc_start,{repeat_1_SUITE,testcase_2b}},
{?eh,tc_done,{repeat_1_SUITE,testcase_2b,ok}},
{?eh,test_stats,{6,0,{0,0}}},
{?eh,tc_start,
{repeat_1_SUITE,{end_per_group,test_group_2,[{repeat,0}]}}},
{repeat_1_SUITE,{end_per_group,test_group_2,[]}}},
{?eh,tc_done,
{repeat_1_SUITE,{end_per_group,test_group_2,[{repeat,0}]},ok}}],
{repeat_1_SUITE,{end_per_group,test_group_2,[]},ok}}],
[{?eh,tc_start,
{repeat_1_SUITE,
{init_per_group,test_group_3,[{repeat_until_all_fail,0}]}}},
{init_per_group,test_group_3,[]}}},
{?eh,tc_done,
{repeat_1_SUITE,
{init_per_group,test_group_3,[{repeat_until_all_fail,0}]},
{init_per_group,test_group_3,[]},
ok}},
{?eh,tc_start,{repeat_1_SUITE,testcase_3a}},
{?eh,tc_done,{repeat_1_SUITE,testcase_3a,ok}},
{?eh,test_stats,{7,0,{0,0}}},
[{?eh,tc_start,
{repeat_1_SUITE,
{init_per_group,test_group_4,[{repeat_until_any_fail,0}]}}},
{init_per_group,test_group_4,[]}}},
{?eh,tc_done,
{repeat_1_SUITE,
{init_per_group,test_group_4,[{repeat_until_any_fail,0}]},
{init_per_group,test_group_4,[]},
ok}},
{?eh,tc_start,{repeat_1_SUITE,testcase_4a}},
{?eh,tc_done,{repeat_1_SUITE,testcase_4a,ok}},
Expand All @@ -250,20 +250,20 @@ test_events(repeat_1) ->
{?eh,test_stats,{9,0,{0,0}}},
{?eh,tc_start,
{repeat_1_SUITE,
{end_per_group,test_group_4,[{repeat_until_any_fail,0}]}}},
{end_per_group,test_group_4,[]}}},
{?eh,tc_done,
{repeat_1_SUITE,
{end_per_group,test_group_4,[{repeat_until_any_fail,0}]},
{end_per_group,test_group_4,[]},
ok}}],
{?eh,tc_start,{repeat_1_SUITE,testcase_3b}},
{?eh,tc_done,{repeat_1_SUITE,testcase_3b,ok}},
{?eh,test_stats,{10,0,{0,0}}},
{?eh,tc_start,
{repeat_1_SUITE,
{end_per_group,test_group_3,[{repeat_until_all_fail,0}]}}},
{end_per_group,test_group_3,[]}}},
{?eh,tc_done,
{repeat_1_SUITE,
{end_per_group,test_group_3,[{repeat_until_all_fail,0}]},
{end_per_group,test_group_3,[]},
ok}}],
{?eh,tc_start,{repeat_1_SUITE,end_per_suite}},
{?eh,tc_done,{repeat_1_SUITE,end_per_suite,ok}},
Expand Down
8 changes: 4 additions & 4 deletions lib/common_test/test/ct_groups_test_2_SUITE_data/groups_1/repeat_1_SUITE.erl
View file
Expand Up @@ -31,12 +31,12 @@ suite() ->

groups() ->
[
{test_group_1, [{repeat,1}], [testcase_1a,testcase_1b]},
{test_group_2, [{repeat,0}], [testcase_2a,testcase_2b]},
{test_group_1, [{repeat,2}], [testcase_1a,testcase_1b]},
{test_group_2, [{repeat,1}], [testcase_2a,testcase_2b]},

{test_group_3, [{repeat_until_all_fail,0}],
{test_group_3, [{repeat_until_all_fail,1}],
[testcase_3a,
{test_group_4, [{repeat_until_any_fail,0}],
{test_group_4, [{repeat_until_any_fail,1}],
[testcase_4a, testcase_4b]},
testcase_3b]}
].
Expand Down
50 changes: 40 additions & 10 deletions lib/test_server/src/test_server_ctrl.erl
View file
Expand Up @@ -1572,7 +1572,7 @@ remove_conf([{conf, _Ref, Props, _MF}|Cases], NoConf, Repeats) ->
case get_repeat(Props) of
undefined ->
remove_conf(Cases, NoConf, Repeats);
{_RepType,0} ->
{_RepType,1} ->
remove_conf(Cases, NoConf, Repeats);
_ ->
remove_conf(Cases, NoConf, true)
Expand Down Expand Up @@ -2248,7 +2248,7 @@ maybe_get_privdir() ->

run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
Config, TimetrapData, Mode, Status) when Type==conf;
Type==make ->
Type==make ->

file:set_cwd(filename:dirname(get(test_server_dir))),
CurrIOHandler = get(test_server_common_io_handler),
Expand Down Expand Up @@ -2490,7 +2490,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
%% will continously update status with test case results
%% without knowing the Ref (but update hd(Status))
{false,new_status(Ref, Status1),Cases1,?void_fun};
{_RepType,0} ->
{_RepType,N} when N =< 1 ->
{false,new_status(Ref, Status1),Cases1,?void_fun};
_ ->
{Copied,_} = copy_cases(Ref, make_ref(), Cs1),
Expand All @@ -2509,7 +2509,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
case RepVal of
undefined ->
{false,EndStatus,Cases1,?void_fun};
{_RepType,0} ->
{_RepType,N} when N =< 1 ->
{false,EndStatus,Cases1,?void_fun};
{repeat,_} ->
{true,EndStatus,CopiedCases++Cases1,?void_fun};
Expand Down Expand Up @@ -2903,9 +2903,9 @@ update_repeat(Props) ->
Props1 =
if N == forever ->
[{RepType,N}|lists:keydelete(RepType, 1, Props)];
N < 2 ->
N < 3 ->
lists:keydelete(RepType, 1, Props);
N >= 2 ->
N >= 3 ->
[{RepType,N-1}|lists:keydelete(RepType, 1, Props)]
end,
%% if shuffle is used in combination with repeat, a new
Expand Down Expand Up @@ -4454,9 +4454,19 @@ collect_cases({conf,InitMF,CaseList,FinF}, St) when is_atom(FinF) ->
collect_cases({conf,InitMF,CaseList,FinMF}, St0) ->
collect_cases({conf,[],InitMF,CaseList,FinMF}, St0);
collect_cases({conf,Props,InitF,CaseList,FinMF}, St) when is_atom(InitF) ->
collect_cases({conf,Props,{St#cc.mod,InitF},CaseList,FinMF}, St);
case init_props(Props) of
{error,_} ->
{ok,[],St};
Props1 ->
collect_cases({conf,Props1,{St#cc.mod,InitF},CaseList,FinMF}, St)
end;
collect_cases({conf,Props,InitMF,CaseList,FinF}, St) when is_atom(FinF) ->
collect_cases({conf,Props,InitMF,CaseList,{St#cc.mod,FinF}}, St);
case init_props(Props) of
{error,_} ->
{ok,[],St};
Props1 ->
collect_cases({conf,Props1,InitMF,CaseList,{St#cc.mod,FinF}}, St)
end;
collect_cases({conf,Props,InitMF,CaseList,FinMF}, St0) ->
case collect_cases(CaseList, St0) of
{ok,[],_St}=Empty ->
Expand All @@ -4468,8 +4478,15 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF}, St0) ->
{ok,[{skip_case,{conf,Ref,InitMF,Comment}} |
FlatCases ++ [{conf,Ref,[],FinMF}]],St};
false ->
{ok,[{conf,Ref,Props,InitMF} |
FlatCases ++ [{conf,Ref,keep_name(Props),FinMF}]],St}
case init_props(Props) of
{error,_} ->
{ok,[],St};
Props1 ->
{ok,[{conf,Ref,Props1,InitMF} |
FlatCases ++ [{conf,Ref,
keep_name(Props1),
FinMF}]],St}
end
end;
{error,_Reason}=Error ->
Error
Expand Down Expand Up @@ -4627,6 +4644,19 @@ in_skip_list({Mod,Func}, [_|SkipList]) ->
in_skip_list(_, []) ->
false.

%% remove unnecessary properties
init_props(Props) ->
case get_repeat(Props) of
Repeat = {_RepType,N} when N < 2 ->
if N == 0 ->
{error,{invalid_property,Repeat}};
true ->
lists:delete(Repeat, Props)
end;
_ ->
Props
end.

keep_name(Props) ->
lists:filter(fun({name,_}) -> true; (_) -> false end, Props).

Expand Down

0 comments on commit 10a06d2

Please sign in to comment.