rebar_ct.erl

%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et
%% -------------------------------------------------------------------
%%
%% rebar: Erlang Build Tools
%%
%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com)
%%
%% Permission is hereby granted, free of charge, to any person obtaining a copy
%% of this software and associated documentation files (the "Software"), to deal
%% in the Software without restriction, including without limitation the rights
%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
%% copies of the Software, and to permit persons to whom the Software is
%% furnished to do so, subject to the following conditions:
%%
%% The above copyright notice and this permission notice shall be included in
%% all copies or substantial portions of the Software.
%%
%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
%% THE SOFTWARE.
%% -------------------------------------------------------------------
%%
%% Targets:
%% test - run common test suites in ./test
%% int_test - run suites in ./int_test
%% perf_test - run suites inm ./perf_test
%%
%% Global options:
%% verbose=1 - show output from the common_test run as it goes
%% suites="foo,bar" - run <test>/foo_SUITE and <test>/bar_SUITE
%% case="mycase" - run individual test case foo_SUITE:mycase
%% -------------------------------------------------------------------
-module(rebar_ct).

-export([ct/2]).

%% for internal use only
-export([info/2]).

-include("rebar.hrl").

%% ===================================================================
%% Public API
%% ===================================================================

ct(Config, File) ->
    TestDir = rebar_config:get_local(Config, ct_dir, "test"),
    LogDir = rebar_config:get_local(Config, ct_log_dir, "logs"),
    run_test_if_present(TestDir, LogDir, Config, File).

%% ===================================================================
%% Internal functions
%% ===================================================================

info(help, ct) ->
    ?CONSOLE(
       "Run common_test suites.~n"
       "~n"
       "Valid rebar.config options:~n"
       "  ~p~n"
       "  ~p~n"
       "  ~p~n"
       "  ~p~n"
       "  ~p~n"
       "Valid command line options:~n"
       "  suites=Suite1,Suite2,...,SuiteN~n"
       "      - run Suite1_SUITE, Suite2_SUITE, ..., SuiteN_SUITE~n"
       "      in the test folder.~n"
       "  groups=Group1,Group2,...,GroupN~n"
       "      - run test groups Group1, Group2, ..., GroupN of specified suites.~n"
       "  cases=Case1,Case2,...,CaseM~n"
       "      - run test cases Case1, Case2, ..., CaseN of specified suites.~n"
       "  case=\"mycase\" - run individual test case Suite1_SUITE:mycase.~n"
       "      This option is deprecated and remains for backward compability.~n"
       "      It is recommended to use 'cases' instead.~n",
       [
        {ct_dir, "itest"},
        {ct_log_dir, "test/logs"},
        {ct_extra_params, "-boot start_sasl -s myapp"},
        {ct_use_short_names, true},
        {ct_search_specs_from_test_dir, false}
       ]).

run_test_if_present(TestDir, LogDir, Config, File) ->
    case filelib:is_dir(TestDir) of
        false ->
            ?WARN("~s directory not present - skipping\n", [TestDir]),
            ok;
        true ->
            case filelib:wildcard(TestDir ++ "/*_SUITE.{beam,erl}") of
                [] ->
                    ?WARN("~s directory present, but no common_test"
                          ++ " SUITES - skipping\n", [TestDir]),
                    ok;
                _ ->
                    try
                        run_test(TestDir, LogDir, Config, File)
                    catch
                        throw:skip ->
                            ok
                    end
            end
    end.

run_test(TestDir, LogDir, Config, _File) ->
    {Cmd, RawLog} = make_cmd(TestDir, LogDir, Config),
    ?DEBUG("ct_run cmd:~n~p~n", [Cmd]),
    clear_log(LogDir, RawLog),
    Output = case rebar_log:is_verbose(Config) of
                 false ->
                     " >> " ++ RawLog ++ " 2>&1";
                 true ->
                 case os:type() of
                   {win32, nt} ->
                     " >> " ++ RawLog ++ " 2>&1";
                   _ ->
                     " 2>&1 | tee -a " ++ RawLog
                 end
             end,

    ShOpts = [{env,[{"TESTDIR", TestDir}]}, return_on_error],
    case rebar_utils:sh(Cmd ++ Output, ShOpts) of
        {ok,_} ->
            %% in older versions of ct_run, this could have been a failure
            %% that returned a non-0 code. Check for that!
            check_success_log(Config, RawLog);
        {error,Res} ->
            %% In newer ct_run versions, this may be a sign of a good compile
            %% that failed cases. In older version, it's a worse error.
            check_fail_log(Config, RawLog, Cmd ++ Output, Res)
    end.

clear_log(LogDir, RawLog) ->
    case filelib:ensure_dir(filename:join(LogDir, "index.html")) of
        ok ->
            NowStr = rebar_utils:now_str(),
            LogHeader = "--- Test run on " ++ NowStr ++ " ---\n",
            ok = file:write_file(RawLog, LogHeader);
        {error, Reason} ->
            ?ERROR("Could not create log dir - ~p\n", [Reason]),
            ?FAIL
    end.

%% calling ct with erl does not return non-zero on failure - have to check
%% log results
check_success_log(Config, RawLog) ->
    check_log(Config, RawLog, fun(Msg) -> ?CONSOLE("DONE.\n~s\n", [Msg]) end).

-type err_handler() :: fun((string()) -> no_return()).
-spec failure_logger(string(), {integer(), string()}) -> err_handler().
failure_logger(Command, {Rc, Output}) ->
    fun(_Msg) ->
            ?ABORT("~s failed with error: ~w and output:~n~s~n",
                   [Command, Rc, Output])
    end.

check_fail_log(Config, RawLog, Command, Result) ->
    check_log(Config, RawLog, failure_logger(Command, Result)).

check_log(Config,RawLogFilename,Fun) ->
    %% read the file and split into a list separated by newlines
    {ok, RawLog} = file:read_file(RawLogFilename),
    Msg = string:tokens(binary_to_list(RawLog), "\n"),
    %% now filter out all the list entries that do not have test
    %% completion strings
    CompleteRuns = lists:filter(fun(M) ->
                                  string:str(M, "TEST COMPLETE") =/= 0
                                end, Msg),
    MakeFailed = lists:filter(fun(M) ->
                                  string:str(M, "{error,make_failed}") =/= 0
                              end, Msg),
    %% the run has failed if at least one of the tests failed
    RunFailed = lists:foldl(fun(M, Acc) ->
                              %% the "0 failed" string must be present for
                              %% the test to be considered successful
                              TestFailed = string:str(M, "0 failed") =:= 0,
                              TestFailed orelse Acc
                            end, false, CompleteRuns),
    if
        MakeFailed =/= [] ->
            show_log(Config, RawLog),
            ?ERROR("Building tests failed\n",[]),
            ?FAIL;

        RunFailed ->
            show_log(Config, RawLog),
            ?ERROR("One or more tests failed\n",[]),
            ?FAIL;

        true ->
            Fun(Msg)
    end.


%% Show the log if it hasn't already been shown because verbose was on
show_log(Config, RawLog) ->
    ?CONSOLE("Showing log\n", []),
    case rebar_log:is_verbose(Config) of
        false ->
            ?CONSOLE("~s", [RawLog]);
        true ->
            ok
    end.

make_cmd(TestDir, RawLogDir, Config) ->
    Cwd = rebar_utils:get_cwd(),
    LogDir = filename:join(Cwd, RawLogDir),
    EbinDir = filename:absname(filename:join(Cwd, "ebin")),
    IncludeDir = filename:join(Cwd, "include"),
    Include = case filelib:is_dir(IncludeDir) of
                  true ->
                      " -include \"" ++ IncludeDir ++ "\"";
                  false ->
                      ""
              end,

    %% Check for the availability of ct_run; if we can't find it, generate a
    %% warning and use the old school, less reliable approach to running CT.
    BaseCmd = case os:find_executable("ct_run") of
                  false ->
                      "erl -noshell -s ct_run script_start -s erlang halt";
                  _ ->
                      "ct_run -noshell"
              end,

    %% Add the code path of the rebar process to the code path. This
    %% includes the dependencies in the code path. The directories
    %% that are part of the root Erlang install are filtered out to
    %% avoid duplication
    R = code:root_dir(),
    NonLibCodeDirs = [P || P <- code:get_path(), not lists:prefix(R, P)],
    CodeDirs = [io_lib:format("\"~s\"", [Dir]) ||
                   Dir <- [EbinDir|NonLibCodeDirs]],
    CodePathString = string:join(CodeDirs, " "),
    Cmd = case get_ct_specs(Config, search_ct_specs_from(Cwd, TestDir, Config)) of
              undefined ->
                  ?FMT("~s"
                       " -pa ~s"
                       " ~s"
                       " ~s"
                       " -logdir \"~s\""
                       " -env TEST_DIR \"~s\"",
                       [BaseCmd,
                        CodePathString,
                        Include,
                        build_name(Config),
                        LogDir,
                        filename:join(Cwd, TestDir)]) ++
                      get_cover_config(Config, Cwd) ++
                      get_ct_config_file(TestDir) ++
                      get_suites(Config, TestDir) ++
                      get_groups(Config) ++
                      get_cases(Config) ++
                      get_extra_params(Config) ++
                      get_config_file(TestDir);
              SpecFlags ->
                  ?FMT("~s"
                       " -pa ~s"
                       " ~s"
                       " ~s"
                       " -logdir \"~s\""
                       " -env TEST_DIR \"~s\"",
                       [BaseCmd,
                        CodePathString,
                        Include,
                        build_name(Config),
                        LogDir,
                        filename:join(Cwd, TestDir)]) ++
                      SpecFlags ++
                      get_cover_config(Config, Cwd) ++
                      get_extra_params(Config)
          end,
    RawLog = filename:join(LogDir, "raw.log"),
    {Cmd, RawLog}.

search_ct_specs_from(Cwd, TestDir, Config) ->
    case rebar_config:get_local(Config, ct_search_specs_from_test_dir, false) of
        true -> filename:join(Cwd, TestDir);
        false ->
          Cwd
    end.

build_name(Config) ->
    %% generate a unique name for our test node, we want
    %% to make sure the odds of name clashing are low
    Random = integer_to_list(crypto:rand_uniform(0, 10000)),
    case rebar_config:get_local(Config, ct_use_short_names, false) of
        true -> "-sname test" ++ Random;
        false -> " -name test" ++ Random ++ "@" ++ net_adm:localhost()
    end.

get_extra_params(Config) ->
    case rebar_config:get_local(Config, ct_extra_params, undefined) of
        undefined ->
            "";
        Defined ->
            " " ++ Defined
    end.

get_ct_specs(Config, Cwd) ->
    case collect_glob(Config, Cwd, ".*\.test\.spec\$") of
        [] -> undefined;
        [Spec] ->
            " -spec " ++ Spec;
        Specs ->
            " -spec " ++
                lists:flatten([io_lib:format("~s ", [Spec]) || Spec <- Specs])
    end.

get_cover_config(Config, Cwd) ->
    case rebar_config:get_local(Config, cover_enabled, false) of
        false ->
            "";
        true ->
            case collect_glob(Config, Cwd, ".*cover\.spec\$") of
                [] ->
                    ?DEBUG("No cover spec found: ~s~n", [Cwd]),
                    "";
                [Spec] ->
                    ?DEBUG("Found cover file ~s~n", [Spec]),
                    " -cover " ++ Spec;
                Specs ->
                    ?ABORT("Multiple cover specs found: ~p~n", [Specs])
            end
    end.

collect_glob(Config, Cwd, Glob) ->
    {true, Deps} = rebar_deps:get_deps_dir(Config),
    DepsDir = filename:basename(Deps),
    CwdParts = filename:split(Cwd),
    filelib:fold_files(
      Cwd,
      Glob,
      true,
      fun(F, Acc) ->
              %% Ignore any specs under the deps/ directory. Do this pulling
              %% the dirname off the F and then splitting it into a list.
              Parts = filename:split(filename:dirname(F)),
              Parts2 = remove_common_prefix(Parts, CwdParts),
              case lists:member(DepsDir, Parts2) of
                  true ->
                      %% There is a directory named "deps" in path
                      Acc;
                  false ->
                      %% No "deps" directory in path
                      [F | Acc]
              end
      end,
      []).

remove_common_prefix([H1|T1], [H1|T2]) ->
    remove_common_prefix(T1, T2);
remove_common_prefix(L1, _) ->
    L1.

get_ct_config_file(TestDir) ->
    Config = filename:join(TestDir, "test.config"),
    case filelib:is_regular(Config) of
        false ->
            " ";
        true ->
            " -ct_config " ++ Config
    end.

get_config_file(TestDir) ->
    Config = filename:join(TestDir, "app.config"),
    case filelib:is_regular(Config) of
        false ->
            " ";
        true ->
            " -erl_args -config " ++ Config
    end.

get_suites(Config, TestDir) ->
    case get_suites(Config) of
        undefined ->
            " -dir " ++ TestDir;
        Suites ->
            Suites1 = [find_suite_path(Suite, TestDir) || Suite <- Suites],
            string:join([" -suite"] ++ Suites1, " ")
    end.

get_suites(Config) ->
    case rebar_config:get_global(Config, suites, undefined) of
        undefined ->
            %% The option 'suite' is deprecated and remains
            %% for backward compatibility.
            %% It is recommended to use 'suites' instead.
            case get_deprecated_global(Config, suite, suites) of
                undefined ->
                    undefined;
                Suite ->
                    [Suite]
            end;
        Suites ->
            string:tokens(Suites, ",")
    end.

find_suite_path(Suite, TestDir) ->
    Path = filename:join(TestDir, Suite ++ "_SUITE.erl"),
    case filelib:is_regular(Path) of
        false ->
            ?WARN("Suite ~s not found\n", [Suite]),
            %% Note - this throw is caught in run_test_if_present/3;
            %% this solution was easier than refactoring the entire module.
            throw(skip);
        true ->
            Path
    end.

get_groups(Config) ->
    case rebar_config:get_global(Config, groups, undefined) of
        undefined ->
            %% The option 'group' was added only for consistency
            %% because there are options 'suite' and 'case'.
            case get_deprecated_global(Config, group, groups) of
                undefined ->
                    "";
                Group ->
                    " -group " ++ Group
            end;
        Groups ->
            Groups1 = string:tokens(Groups, ","),
            string:join([" -group"] ++ Groups1, " ")
    end.

get_cases(Config) ->
    case rebar_config:get_global(Config, cases, undefined) of
        undefined ->
            %% The option 'case' is deprecated and remains
            %% for backward compatibility.
            %% It is recommended to use 'cases' instead.
            case get_deprecated_global(Config, 'case', cases) of
                undefined ->
                    "";
                Case ->
                    " -case " ++ Case
            end;
        Cases ->
            Cases1 = string:tokens(Cases, ","),
            string:join([" -case"] ++ Cases1, " ")
    end.

get_deprecated_global(Config, OldOpt, NewOpt) ->
    rebar_utils:get_deprecated_global(
      Config, OldOpt, NewOpt, undefined, "in the future").