From 7640a73a09f3bb8f6133b537e5630cf3dbb24536 Mon Sep 17 00:00:00 2001 From: Johannes Christ Date: Sat, 11 Nov 2023 17:18:34 +0100 Subject: [PATCH] ct: Allow empty groups in test suites The logic to trim empty test cases was moved from the find groups function (which will now also return groups with no testcases) to the expand groups function. Groups with no testcases will not be run (e.g. their init_per_group and end_per_group functions will not be called), but no error will be thrown as previously. Fixes #4362. --- lib/common_test/src/ct_groups.erl | 14 ++- .../test/ct_groups_search_SUITE.erl | 33 ++++++- lib/common_test/test/ct_misc_1_SUITE.erl | 91 +++++++++++++++---- .../empty_group_SUITE.erl | 10 ++ 4 files changed, 125 insertions(+), 23 deletions(-) create mode 100644 lib/common_test/test/ct_misc_1_SUITE_data/empty_group_SUITE.erl diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl index 2c4b7a383e94..b6cc094fbf19 100644 --- a/lib/common_test/src/ct_groups.erl +++ b/lib/common_test/src/ct_groups.erl @@ -66,6 +66,17 @@ find_groups1(Mod, GrNames, TCs, GroupDefs) -> GroupDefs, FindAll), [Conf || Conf <- Found, Conf /= 'NOMATCH']. +%% Finds out if the given group, or child cases / child group of a group, is non-empty. +not_empty_conf({conf, _Props, _Init, Children, _End}) -> not_empty_conf(Children); +not_empty_conf([] = _Cases) -> false; +not_empty_conf([_ | _] = _Cases) -> true; +not_empty_conf(Name) when is_atom(Name) -> true; +% Other, unexpected formats we don't actively care about, such as `{mnesia_evil_backup, all}`. +not_empty_conf(_Format) -> true. + +elide_empty_confs(Confs) -> + lists:filtermap(fun not_empty_conf/1, Confs). + %% Locate all groups find(Mod, all, all, [{Name,Props,Tests} | Gs], Known, Defs, _) when is_atom(Name), is_list(Props), is_list(Tests) -> @@ -253,7 +264,6 @@ find(_Mod, _GrNames, _TCs, [], _Known, _Defs, _) -> trim({conf,Props,Init,Tests,End}) -> try trim(Tests) of - [] -> []; Tests1 -> [{conf,Props,Init,Tests1,End}] catch throw:_ -> [] @@ -499,7 +509,7 @@ make_conf(Mod, Name, Props, TestSpec) -> %%%----------------------------------------------------------------- expand_groups([H | T], ConfTests, Mod) -> - [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]; + elide_empty_confs([expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]); expand_groups([], _ConfTests, _Mod) -> []; expand_groups({group,Name}, ConfTests, Mod) -> diff --git a/lib/common_test/test/ct_groups_search_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE.erl index 13742c11b12c..0d4ff9a8d9b1 100644 --- a/lib/common_test/test/ct_groups_search_SUITE.erl +++ b/lib/common_test/test/ct_groups_search_SUITE.erl @@ -127,7 +127,8 @@ groups() -> testcase_in_sub_groups11, testcase_in_sub_groups12, testcase_in_sub_groups13, - bad_testcase_in_sub_groups1]}, + bad_testcase_in_sub_groups1, + bad_testcase_in_sub_groups2]}, {run_groups,[sequence],[run_groups_with_options, run_groups_with_testspec]} @@ -488,6 +489,10 @@ testcase_in_top_groups3(_) -> [{conf,[{name,top1}], {?M2,init_per_group}, [{?M2,top1_tc1}], + {?M2,end_per_group}}, + {conf,[{name,top2}], + {?M2,init_per_group}, + [], {?M2,end_per_group}}] = Found, {?M2,GPath,TCs,Found}. @@ -499,7 +504,11 @@ testcase_in_top_groups4(_) -> Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), - [{conf,[{name,top2}], + [{conf,[{name,top1}], + {?M2,init_per_group}, + [], + {?M2,end_per_group}}, + {conf,[{name,top2}], {?M2,init_per_group}, [{conf,[{name,sub21}], {?M2,init_per_group}, @@ -527,7 +536,11 @@ testcase_in_top_groups5(_) -> Found = ct_groups:find_groups(?M2, [top1,top2], [sub21_tc1,sub22_tc1], groups2()), - [{conf,[{name,top2}], + [{conf,[{name,top1}], + {?M2,init_per_group}, + [], + {?M2,end_per_group}}, + {conf,[{name,top2}], {?M2,init_per_group}, [{conf,[{name,sub21}], {?M2,init_per_group}, @@ -964,7 +977,10 @@ bad_testcase_in_sub_groups1(_) -> Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), - [] = Found, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [], + {?M2,end_per_group}}] = Found, {?M2,GPath,TCs,Found}. @@ -975,7 +991,10 @@ bad_testcase_in_sub_groups2(_) -> Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), - [] = Found, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [], + {?M2,end_per_group}}] = Found, {?M2,GPath,TCs,Found}. @@ -1208,6 +1227,10 @@ test_events(run_groups_with_testspec, Params, Events) -> flatten_tests({conf,[{name,G}|_],{Mod,_I},Tests,_E}) -> lists:flatten([{group,Mod,G} | flatten_tests(Tests)]); +flatten_tests([{conf,[{name,G}|_],{Mod,_I},[],_E} | Confs]) -> + % Elide empty groups, like ct_groups does, to prevent + % trying to verify their non-existent events. + flatten_tests(Confs); flatten_tests([{conf,[{name,G}|_],{Mod,_I},Tests,_E} | Confs]) -> lists:flatten([{group,Mod,G} | flatten_tests(Tests)]) ++ lists:flatten(flatten_tests(Confs)); diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl index f1d783fc8da5..822410eced62 100644 --- a/lib/common_test/test/ct_misc_1_SUITE.erl +++ b/lib/common_test/test/ct_misc_1_SUITE.erl @@ -60,7 +60,7 @@ end_per_testcase(TestCase, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [beam_me_up, {group,parse_table}, groups_bad_1]. + [beam_me_up, {group,parse_table}, groups_bad_1, empty_group]. groups() -> [{parse_table,[parallel], @@ -177,22 +177,16 @@ parse_table_one_column_multiline(Config) when is_list(Config) -> groups_bad_1(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), Suite = filename:join(DataDir, "bad_groups_SUITE"), - {Opts,ERPid} = setup([{suite,Suite}, - {label,groups_bad_1}], Config), + verify_events(?FUNCTION_NAME, Suite, Config). - ok = ct_test_support:run(Opts, Config), - Events = ct_test_support:get_events(ERPid, Config), - - ct_test_support:log_events(bad_groups, - reformat(Events, ?eh), - ?config(priv_dir, Config), - Opts), - - TestEvents = test_events(groups_bad_1), - ok = ct_test_support:verify_events(TestEvents, Events, Config). +%%%----------------------------------------------------------------- +%%% +empty_group(Config) when is_list(Config) -> + DataDir = proplists:get_value(data_dir, Config), + Suite = filename:join(DataDir, "empty_group_SUITE"), + verify_events(?FUNCTION_NAME, Suite, Config). - %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -207,8 +201,19 @@ setup(Test, Config) -> reformat(Events, EH) -> ct_test_support:reformat(Events, EH). -%reformat(Events, _EH) -> -% Events. + +verify_events(Case, Suite, Config) -> + {Opts, ERPid} = setup([{suite, Suite}, {label, Case}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Case, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = test_events(Case), + ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- %%% TEST EVENTS @@ -277,4 +282,58 @@ test_events(groups_bad_1) -> 'Invalid reference to group unexist in bad_groups_SUITE:all/0'}}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} + ]; + +test_events(empty_group) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + [{?eh,tc_start, + {ct_framework, + {init_per_group,one_testcase,[{suite,empty_group_SUITE}]}}}, + {?eh,tc_done, + {ct_framework, + {init_per_group,one_testcase,[{suite,empty_group_SUITE}]}, + ok}}, + {?eh,tc_start,{empty_group_SUITE,t1}}, + {?eh,tc_done,{empty_group_SUITE,t1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start, + {ct_framework,{end_per_group,one_testcase,[{suite,empty_group_SUITE}]}}}, + {?eh,tc_done, + {ct_framework, + {end_per_group,one_testcase,[{suite,empty_group_SUITE}]}, + ok}}], + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + [{?eh,tc_start, + {ct_framework, + {init_per_group,one_testcase,[{suite,empty_group_SUITE}]}}}, + {?eh,tc_done, + {ct_framework, + {init_per_group,one_testcase,[{suite,empty_group_SUITE}]}, + ok}}, + {?eh,tc_start,{empty_group_SUITE,t1}}, + {?eh,tc_done,{empty_group_SUITE,t1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start, + {ct_framework,{end_per_group,one_testcase,[{suite,empty_group_SUITE}]}}}, + {?eh,tc_done, + {ct_framework, + {end_per_group,one_testcase,[{suite,empty_group_SUITE}]}, + ok}}], + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_misc_1_SUITE_data/empty_group_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE_data/empty_group_SUITE.erl new file mode 100644 index 000000000000..66cc587f2b19 --- /dev/null +++ b/lib/common_test/test/ct_misc_1_SUITE_data/empty_group_SUITE.erl @@ -0,0 +1,10 @@ +-module(empty_group_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +all() -> [{group, one_testcase}, {group, zero_testcases}]. + +groups() -> [{one_testcase, [t1]}, {zero_testcases, []}]. + +t1(_Config) -> + ok.