ct: Allow empty groups in test suites

The logic to trim empty test cases was moved from the find groups function (which will now also return groups with no testcases) to the expand groups function. Groups with no testcases will not be run (e.g. their init_per_group and end_per_group functions will not be called), but no error will be thrown as previously. Fixes erlang#4362.
jchristgit · Nov 30, 2023 · 7640a73 · 7640a73
1 parent 0f1e509
commit 7640a73
Show file tree

Hide file tree

Showing 4 changed files with 125 additions and 23 deletions.
diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl
@@ -66,6 +66,17 @@ find_groups1(Mod, GrNames, TCs, GroupDefs) ->
 		 GroupDefs, FindAll),
     [Conf || Conf <- Found, Conf /= 'NOMATCH'].
 
+%% Finds out if the given group, or child cases / child group of a group, is non-empty.
+not_empty_conf({conf, _Props, _Init, Children, _End}) -> not_empty_conf(Children);
+not_empty_conf([] = _Cases) -> false;
+not_empty_conf([_ | _] = _Cases) -> true;
+not_empty_conf(Name) when is_atom(Name) -> true;
+% Other, unexpected formats we don't actively care about, such as `{mnesia_evil_backup, all}`.
+not_empty_conf(_Format) -> true.
+
+elide_empty_confs(Confs) ->
+    lists:filtermap(fun not_empty_conf/1, Confs).
+
 %% Locate all groups
 find(Mod, all, all, [{Name,Props,Tests} | Gs], Known, Defs, _) 
   when is_atom(Name), is_list(Props), is_list(Tests) ->
@@ -253,7 +264,6 @@ find(_Mod, _GrNames, _TCs, [], _Known, _Defs, _) ->
 
 trim({conf,Props,Init,Tests,End}) ->
     try trim(Tests) of
-	[] -> [];
 	Tests1 -> [{conf,Props,Init,Tests1,End}]
     catch
 	throw:_ -> []
@@ -499,7 +509,7 @@ make_conf(Mod, Name, Props, TestSpec) ->
 %%%-----------------------------------------------------------------
 
 expand_groups([H | T], ConfTests, Mod) ->
-    [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)];
+    elide_empty_confs([expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]);
 expand_groups([], _ConfTests, _Mod) ->
     [];
 expand_groups({group,Name}, ConfTests, Mod) ->

diff --git a/lib/common_test/test/ct_groups_search_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE.erl
@@ -127,7 +127,8 @@ groups() ->
 		      testcase_in_sub_groups11,
 		      testcase_in_sub_groups12,
 		      testcase_in_sub_groups13,
-		      bad_testcase_in_sub_groups1]},
+		      bad_testcase_in_sub_groups1,
+		      bad_testcase_in_sub_groups2]},
 
      {run_groups,[sequence],[run_groups_with_options,
 			     run_groups_with_testspec]}
@@ -488,6 +489,10 @@ testcase_in_top_groups3(_) ->
     [{conf,[{name,top1}],
       {?M2,init_per_group},
       [{?M2,top1_tc1}],
+      {?M2,end_per_group}},
+     {conf,[{name,top2}],
+      {?M2,init_per_group},
+      [],
       {?M2,end_per_group}}] = Found,
 
     {?M2,GPath,TCs,Found}.
@@ -499,7 +504,11 @@ testcase_in_top_groups4(_) ->
 
     Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
 
-    [{conf,[{name,top2}],
+    [{conf,[{name,top1}],
+      {?M2,init_per_group},
+      [],
+      {?M2,end_per_group}},
+     {conf,[{name,top2}],
       {?M2,init_per_group},
       [{conf,[{name,sub21}],
 	{?M2,init_per_group},
@@ -527,7 +536,11 @@ testcase_in_top_groups5(_) ->
     Found = ct_groups:find_groups(?M2, [top1,top2], [sub21_tc1,sub22_tc1],
 				  groups2()),
 
-    [{conf,[{name,top2}],
+    [{conf,[{name,top1}],
+      {?M2,init_per_group},
+      [],
+      {?M2,end_per_group}},
+     {conf,[{name,top2}],
       {?M2,init_per_group},
       [{conf,[{name,sub21}],
 	{?M2,init_per_group},
@@ -964,7 +977,10 @@ bad_testcase_in_sub_groups1(_) ->
 
     Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
 
-    [] = Found,
+    [{conf,[{name,sub2xx}],
+      {?M2,init_per_group},
+      [],
+      {?M2,end_per_group}}] = Found,
 
     {?M2,GPath,TCs,Found}.
 
@@ -975,7 +991,10 @@ bad_testcase_in_sub_groups2(_) ->
 
     Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
 
-    [] = Found,
+    [{conf,[{name,sub2xx}],
+      {?M2,init_per_group},
+      [],
+      {?M2,end_per_group}}] = Found,
 
     {?M2,GPath,TCs,Found}.
 
@@ -1208,6 +1227,10 @@ test_events(run_groups_with_testspec, Params, Events) ->
 
 flatten_tests({conf,[{name,G}|_],{Mod,_I},Tests,_E}) ->
     lists:flatten([{group,Mod,G} | flatten_tests(Tests)]);
+flatten_tests([{conf,[{name,G}|_],{Mod,_I},[],_E} | Confs]) ->
+    % Elide empty groups, like ct_groups does, to prevent
+    % trying to verify their non-existent events.
+    flatten_tests(Confs);
 flatten_tests([{conf,[{name,G}|_],{Mod,_I},Tests,_E} | Confs]) ->
     lists:flatten([{group,Mod,G} | flatten_tests(Tests)]) ++
 	lists:flatten(flatten_tests(Confs));

diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl
@@ -60,7 +60,7 @@ end_per_testcase(TestCase, Config) ->
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [beam_me_up, {group,parse_table}, groups_bad_1].
+    [beam_me_up, {group,parse_table}, groups_bad_1, empty_group].
 
 groups() -> 
     [{parse_table,[parallel], 
@@ -177,22 +177,16 @@ parse_table_one_column_multiline(Config) when is_list(Config) ->
 groups_bad_1(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
     Suite = filename:join(DataDir, "bad_groups_SUITE"),
-    {Opts,ERPid} = setup([{suite,Suite},
-      {label,groups_bad_1}], Config),
+    verify_events(?FUNCTION_NAME, Suite, Config).
 
-  ok = ct_test_support:run(Opts, Config),
-  Events = ct_test_support:get_events(ERPid, Config),
-
-  ct_test_support:log_events(bad_groups,
-           reformat(Events, ?eh),
-           ?config(priv_dir, Config),
-           Opts),
-
-  TestEvents = test_events(groups_bad_1),
-  ok = ct_test_support:verify_events(TestEvents, Events, Config).
+%%%-----------------------------------------------------------------
+%%%
 
+empty_group(Config) when is_list(Config) ->
+    DataDir = proplists:get_value(data_dir, Config),
+    Suite = filename:join(DataDir, "empty_group_SUITE"),
+    verify_events(?FUNCTION_NAME, Suite, Config).
 
-
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
 %%%-----------------------------------------------------------------
@@ -207,8 +201,19 @@ setup(Test, Config) ->
 
 reformat(Events, EH) ->
     ct_test_support:reformat(Events, EH).
-%reformat(Events, _EH) ->
-%    Events.
+
+verify_events(Case, Suite, Config) ->
+    {Opts, ERPid} = setup([{suite, Suite}, {label, Case}], Config),
+    ok = ct_test_support:run(Opts, Config),
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(Case,
+             reformat(Events, ?eh),
+             ?config(priv_dir, Config),
+             Opts),
+
+    TestEvents = test_events(Case),
+    ok = ct_test_support:verify_events(TestEvents, Events, Config).
 
 %%%-----------------------------------------------------------------
 %%% TEST EVENTS
@@ -277,4 +282,58 @@ test_events(groups_bad_1) ->
                     'Invalid reference to group unexist in bad_groups_SUITE:all/0'}}}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
+    ];
+
+test_events(empty_group) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,1}},
+     {?eh,tc_start,{ct_framework,init_per_suite}},
+     {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+     [{?eh,tc_start,
+          {ct_framework,
+              {init_per_group,one_testcase,[{suite,empty_group_SUITE}]}}},
+      {?eh,tc_done,
+          {ct_framework,
+              {init_per_group,one_testcase,[{suite,empty_group_SUITE}]},
+              ok}},
+       {?eh,tc_start,{empty_group_SUITE,t1}},
+       {?eh,tc_done,{empty_group_SUITE,t1,ok}},
+       {?eh,test_stats,{1,0,{0,0}}},
+      {?eh,tc_start,
+          {ct_framework,{end_per_group,one_testcase,[{suite,empty_group_SUITE}]}}},
+      {?eh,tc_done,
+          {ct_framework,
+              {end_per_group,one_testcase,[{suite,empty_group_SUITE}]},
+              ok}}],
+     {?eh,tc_start,{ct_framework,end_per_suite}},
+     {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]},
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,1}},
+     {?eh,tc_start,{ct_framework,init_per_suite}},
+     {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+     [{?eh,tc_start,
+          {ct_framework,
+              {init_per_group,one_testcase,[{suite,empty_group_SUITE}]}}},
+      {?eh,tc_done,
+          {ct_framework,
+              {init_per_group,one_testcase,[{suite,empty_group_SUITE}]},
+              ok}},
+       {?eh,tc_start,{empty_group_SUITE,t1}},
+       {?eh,tc_done,{empty_group_SUITE,t1,ok}},
+       {?eh,test_stats,{1,0,{0,0}}},
+      {?eh,tc_start,
+          {ct_framework,{end_per_group,one_testcase,[{suite,empty_group_SUITE}]}}},
+      {?eh,tc_done,
+          {ct_framework,
+              {end_per_group,one_testcase,[{suite,empty_group_SUITE}]},
+              ok}}],
+     {?eh,tc_start,{ct_framework,end_per_suite}},
+     {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
     ].
diff --git a/lib/common_test/test/ct_misc_1_SUITE_data/empty_group_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE_data/empty_group_SUITE.erl
@@ -0,0 +1,10 @@
+-module(empty_group_SUITE).
+-compile(export_all).
+-compile(nowarn_export_all).
+
+all() -> [{group, one_testcase}, {group, zero_testcases}].
+
+groups() -> [{one_testcase, [t1]}, {zero_testcases, []}].
+
+t1(_Config) ->
+  ok.