Permalink
Browse files

Merge branch 'siri/common_test/force_stop-skip_rest/OTP-10856' into m…

…aint

* siri/common_test/force_stop-skip_rest/OTP-10856:
  [common_test] Document '-force_stop skip_rest' option to ct_run
  [common_test] Add tests for repeated testruns
  [common_test] Add support for testing repeated testruns
  [common_test] Add -force_stop skip_rest option when repeating tests
  • Loading branch information...
sirihansen committed Apr 4, 2013
2 parents e72043e + 789f7e8 commit c3b26ff8c9d79ebee20fe33b0d8954e5cc68a29f
@@ -113,9 +113,9 @@
[-muliply_timetraps Multiplier]
[-scale_timetraps]
[-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]
- [-repeat N [-force_stop]] |
- [-duration HHMMSS [-force_stop]] |
- [-until [YYMoMoDD]HHMMSS [-force_stop]]
+ [-repeat N] |
+ [-duration HHMMSS [-force_stop [skip_rest]]] |
+ [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]
[-basic_html]
[-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and ..
CTHModuleN CTHOptsN]
@@ -149,9 +149,9 @@
[-muliply_timetraps Multiplier]
[-scale_timetraps]
[-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]
- [-repeat N [-force_stop]] |
- [-duration HHMMSS [-force_stop]] |
- [-until [YYMoMoDD]HHMMSS [-force_stop]]
+ [-repeat N] |
+ [-duration HHMMSS [-force_stop [skip_rest]]] |
+ [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]
[-basic_html]
[-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and ..
CTHModuleN CTHOptsN]
@@ -174,7 +174,7 @@
<item><c><![CDATA[-repeat <n>]]></c>, tells Common Test to repeat the tests n times (see below).</item>
<item><c><![CDATA[-duration <time>]]></c>, tells Common Test to repeat the tests for duration of time (see below).</item>
<item><c><![CDATA[-until <stop_time>]]></c>, tells Common Test to repeat the tests until stop_time (see below).</item>
- <item><c>-force_stop</c>, on timeout, the test run will be aborted when current test job is finished (see below).</item>
+ <item><c>-force_stop [skip_rest]</c>, on timeout, the test run will be aborted when current test job is finished. If <c>skip_rest</c> is provided the rest of the test cases in the current test job will be skipped (see below).</item>
<item><c><![CDATA[-decrypt_key <key>]]></c>, provides a decryption key for
<seealso marker="config_file_chapter#encrypted_config_files">encrypted configuration files</seealso>.</item>
<item><c><![CDATA[-decrypt_file <key_file>]]></c>, points out a file containing a decryption key for
@@ -1273,6 +1273,7 @@
<item><c>-duration DurTime ({duration,DurTime})</c>, where <c>DurTime</c> is the duration, see below.</item>
<item><c>-until StopTime ({until,StopTime})</c>, where <c>StopTime</c> is finish time, see below.</item>
<item><c>-force_stop ({force_stop,true})</c></item>
+ <item><c>-force_stop skip_rest ({force_stop,skip_rest})</c></item>
</list>
<p>The duration time, <c>DurTime</c>, is specified as <c>HHMMSS</c>. Example:
<c>-duration 012030</c> or <c>{duration,"012030"}</c>, means the tests will
@@ -1283,12 +1284,16 @@
Example: <c>-until 071001120000</c> or <c>{until,"071001120000"}</c>, which means the tests
will be executed and (if time allows) repeated, until 12 o'clock on the 1st of Oct 2007.</p>
- <p>When timeout occurs, Common Test will never abort the test run immediately, since
+ <p>When timeout occurs, Common Test will never abort the ongoing test case, since
this might leave the system under test in an undefined, and possibly bad, state.
- Instead Common Test will finish the current test job, or the complete test
- run, before stopping. The latter is the default behaviour. The <c>force_stop</c>
- flag/option tells Common Test to stop as soon as the current test job is finished.
- Note that since Common Test always finishes off the current test job or test session,
+ Instead Common Test will by default finish the current test
+ run before stopping. If the <c>force_stop</c> flag is
+ given, Common Test will stop as soon as the current test job
+ is finished, and if the <c>force_stop</c> flag is given with
+ <c>skip_rest</c> Common Test will only complete the current
+ test case and skip the rest of the tests in the test job.
+ Note that since Common Test always finishes off at least the
+ current test case,
the time specified with <c>duration</c> or <c>until</c> is never definitive!</p>
<p>Log files from every single repeated test run is saved in normal Common Test fashion (see above).
@@ -1311,6 +1316,18 @@
aborted after the to1 test and before the to2 test.</p>
<p>Example 2:</p>
+ <pre>
+ $ ct_run -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -duration 001000 -forces_stop skip_rest</pre>
+ <p>Here the same test run as in Example 1, but with the
+ <c>force_stop</c> flag set to <c>skip_rest</c>. If the timeout
+ occurs while executing tests in directory to1, the rest of the
+ test cases in to1 will be skipped and then the test will be
+ aborted without running the tests in to2 another time. If the
+ timeout occurs while executing tests in directory to2, then the
+ rest of the test cases in to2 will be skipped and then the test
+ will be aborted.</p>
+
+ <p>Example 3:</p>
<pre>
$ date
Fri Sep 28 15:00:00 MEST 2007
@@ -1321,7 +1338,7 @@
Common Test will finish the entire test run before stopping (i.e. the to1 and to2 test
will always both be executed in the same test run).</p>
- <p>Example 3:</p>
+ <p>Example 4:</p>
<pre>
$ ct_run -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -repeat 5</pre>
<p>Here the test run, including both the to1 and the to2 test, will be repeated 5 times.</p>
@@ -153,7 +153,7 @@ run(TestDirs) ->
%%% {auto_compile,Bool} | {create_priv_dir,CreatePrivDir} |
%%% {multiply_timetraps,M} | {scale_timetraps,Bool} |
%%% {repeat,N} | {duration,DurTime} | {until,StopTime} |
-%%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
+%%% {force_stop,ForceStop} | {decrypt,DecryptKeyOrFile} |
%%% {refresh_logs,LogDir} | {logopts,LogOpts} |
%%% {verbosity,VLevels} | {basic_html,Bool} |
%%% {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} |
@@ -184,6 +184,7 @@ run(TestDirs) ->
%%% N = integer()
%%% DurTime = string(HHMMSS)
%%% StopTime = string(YYMoMoDDHHMMSS) | string(HHMMSS)
+%%% ForceStop = skip_rest | Bool
%%% DecryptKeyOrFile = {key,DecryptKey} | {file,DecryptFile}
%%% DecryptKey = string()
%%% DecryptFile = string()
@@ -64,38 +64,46 @@ init_tc(Mod,Func,Config) ->
ok
end,
- case ct_util:get_testdata(curr_tc) of
- {Suite,{suite0_failed,{require,Reason}}} ->
- {skip,{require_failed_in_suite0,Reason}};
- {Suite,{suite0_failed,_}=Failure} ->
- {skip,Failure};
+ case Func=/=end_per_suite
+ andalso Func=/=end_per_group
+ andalso ct_util:get_testdata(skip_rest) of
+ true ->
+ {skip,"Repeated test stopped by force_stop option"};
_ ->
- ct_util:update_testdata(curr_tc,
- fun(undefined) ->
- [{Suite,Func}];
- (Running) ->
- [{Suite,Func}|Running]
- end, [create]),
- case ct_util:read_suite_data({seq,Suite,Func}) of
- undefined ->
- init_tc1(Mod,Suite,Func,Config);
- Seq when is_atom(Seq) ->
- case ct_util:read_suite_data({seq,Suite,Seq}) of
- [Func|TCs] -> % this is the 1st case in Seq
- %% make sure no cases in this seq are
- %% marked as failed from an earlier execution
- %% in the same suite
- lists:foreach(
- fun(TC) ->
- ct_util:save_suite_data({seq,Suite,TC},
- Seq)
- end, TCs);
- _ ->
- ok
- end,
- init_tc1(Mod,Suite,Func,Config);
- {failed,Seq,BadFunc} ->
- {skip,{sequence_failed,Seq,BadFunc}}
+ case ct_util:get_testdata(curr_tc) of
+ {Suite,{suite0_failed,{require,Reason}}} ->
+ {skip,{require_failed_in_suite0,Reason}};
+ {Suite,{suite0_failed,_}=Failure} ->
+ {skip,Failure};
+ _ ->
+ ct_util:update_testdata(curr_tc,
+ fun(undefined) ->
+ [{Suite,Func}];
+ (Running) ->
+ [{Suite,Func}|Running]
+ end, [create]),
+ case ct_util:read_suite_data({seq,Suite,Func}) of
+ undefined ->
+ init_tc1(Mod,Suite,Func,Config);
+ Seq when is_atom(Seq) ->
+ case ct_util:read_suite_data({seq,Suite,Seq}) of
+ [Func|TCs] -> % this is the 1st case in Seq
+ %% make sure no cases in this seq are
+ %% marked as failed from an earlier execution
+ %% in the same suite
+ lists:foreach(
+ fun(TC) ->
+ ct_util:save_suite_data(
+ {seq,Suite,TC},
+ Seq)
+ end, TCs);
+ _ ->
+ ok
+ end,
+ init_tc1(Mod,Suite,Func,Config);
+ {failed,Seq,BadFunc} ->
+ {skip,{sequence_failed,Seq,BadFunc}}
+ end
end
end.
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2007-2012. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2013. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -23,7 +23,7 @@
%%% start flags (or equivalent ct:run_test/1 options) are supported:
%%% -until <StopTime>, StopTime = YYMoMoDDHHMMSS | HHMMSS
%%% -duration <DurTime>, DurTime = HHMMSS
-%%% -force_stop
+%%% -force_stop [skip_rest]
%%% -repeat <N>, N = integer()</p>
-module(ct_repeat).
@@ -62,12 +62,15 @@ loop_test(If,Args) when is_list(Args) ->
io:format("\nCommon Test: "
"Will repeat tests for ~s.\n\n",[ts(Secs)]),
TPid =
- case lists:keymember(force_stop,1,Args) of
- true ->
+ case proplists:get_value(force_stop,Args) of
+ False when False==false; False==undefined ->
+ undefined;
+ ForceStop ->
CtrlPid = self(),
- spawn(fun() -> stop_after(CtrlPid,Secs) end);
- false ->
- undefined
+ spawn(
+ fun() ->
+ stop_after(CtrlPid,Secs,ForceStop)
+ end)
end,
Args1 = [{loop_info,[{stop_time,Secs,StopTime,1}]} | Args],
loop(If,stop_time,0,Secs,StopTime,Args1,TPid,[])
@@ -212,7 +215,7 @@ get_stop_time(until,[Y1,Y2,Mo1,Mo2,D1,D2,H1,H2,Mi1,Mi2,S1,S2]) ->
list_to_integer([S1,S2])},
calendar:datetime_to_gregorian_seconds({Date,Time});
-get_stop_time(until,Time) ->
+get_stop_time(until,Time=[_,_,_,_,_,_]) ->
get_stop_time(until,"000000"++Time);
get_stop_time(duration,[H1,H2,Mi1,Mi2,S1,S2]) ->
@@ -227,10 +230,17 @@ cancel(Pid) ->
%% After Secs, abort will make the test_server finish the current
%% job, then empty the job queue and stop.
-stop_after(_CtrlPid,Secs) ->
+stop_after(_CtrlPid,Secs,ForceStop) ->
timer:sleep(Secs*1000),
+ case ForceStop of
+ SkipRest when SkipRest==skip_rest; SkipRest==["skip_rest"] ->
+ ct_util:set_testdata({skip_rest,true});
+ _ ->
+ ok
+ end,
test_server_ctrl:abort().
+
%% Callback from ct_run to print loop info to system log.
log_loop_info(Args) ->
case lists:keysearch(loop_info,1,Args) of
@@ -259,11 +269,11 @@ log_loop_info(Args) ->
io_lib:format("Test time remaining: ~w secs (~w%)\n",
[Secs,trunc((Secs/Secs0)*100)]),
LogStr4 =
- case lists:keymember(force_stop,1,Args) of
- true ->
- io_lib:format("force_stop is enabled",[]);
- _ ->
- ""
+ case proplists:get_value(force_stop,Args) of
+ False when False==false; False==undefined ->
+ "";
+ ForceStop ->
+ io_lib:format("force_stop is set to: ~w",[ForceStop])
end,
ct_logs:log("Test loop info",LogStr1++LogStr2++LogStr3++LogStr4,[])
end.
@@ -771,9 +771,9 @@ script_usage() ->
"\n\t[-scale_timetraps]"
"\n\t[-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]"
"\n\t[-basic_html]"
- "\n\t[-repeat N [-force_stop]] |"
- "\n\t[-duration HHMMSS [-force_stop]] |"
- "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
+ "\n\t[-repeat N] |"
+ "\n\t[-duration HHMMSS [-force_stop [skip_rest]]] |"
+ "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]\n\n"),
io:format("Run tests using test specification:\n\n"
"\tct_run -spec TestSpec1 TestSpec2 .. TestSpecN"
"\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
@@ -795,9 +795,9 @@ script_usage() ->
"\n\t[-scale_timetraps]"
"\n\t[-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]"
"\n\t[-basic_html]"
- "\n\t[-repeat N [-force_stop]] |"
- "\n\t[-duration HHMMSS [-force_stop]] |"
- "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
+ "\n\t[-repeat N] |"
+ "\n\t[-duration HHMMSS [-force_stop [skip_rest]]] |"
+ "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]\n\n"),
io:format("Refresh the HTML index files:\n\n"
"\tct_run -refresh_logs [LogDir]"
"[-logdir LogDir] "
@@ -2933,6 +2933,8 @@ opts2args(EnvStartOpts) ->
[];
({create_priv_dir,PD}) when is_atom(PD) ->
[{create_priv_dir,[atom_to_list(PD)]}];
+ ({force_stop,skip_rest}) ->
+ [{force_stop,["skip_rest"]}];
({force_stop,true}) ->
[{force_stop,[]}];
({force_stop,false}) ->
@@ -38,6 +38,7 @@ MODULES= \
ct_groups_spec_SUITE \
ct_sequence_1_SUITE \
ct_repeat_1_SUITE \
+ ct_repeat_testrun_SUITE \
ct_testspec_1_SUITE \
ct_testspec_2_SUITE \
ct_testspec_3_SUITE \
Oops, something went wrong.

0 comments on commit c3b26ff

Please sign in to comment.