From 78ed642967c358520fdd7bcbef764a7191335897 Mon Sep 17 00:00:00 2001 From: Jared Morrow Date: Sun, 21 Jun 2015 22:23:49 -0600 Subject: [PATCH 01/89] Add license to relnotes tool per reviewer request --- pr2relnotes.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pr2relnotes.py b/pr2relnotes.py index 6e9a4f49..c58920cf 100755 --- a/pr2relnotes.py +++ b/pr2relnotes.py @@ -1,5 +1,27 @@ #!/usr/bin/env python +# The MIT License (MIT) +# +# Copyright (c) 2015 Jared Morrow (github.com/jaredmorrow) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + ## Install info ## $ virtualenv env ## $ source env/bin/activate From a47230899e879b1d49ca5ee8b33e1f0f34fd6a61 Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Thu, 25 Jun 2015 14:05:46 +0100 Subject: [PATCH 02/89] Generate reserved file for erlc regression test Reserved files such as ._do_not_compile.erl should be generated automatically when running the regression test and not added to the repo. .tar.gz source releases are not correctly generated when handling such files. --- inttest/erlc/erlc_rt.erl | 4 ++++ inttest/erlc/src/._do_not_compile.erl | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 inttest/erlc/src/._do_not_compile.erl diff --git a/inttest/erlc/erlc_rt.erl b/inttest/erlc/erlc_rt.erl index 7e0c5dc4..0c1f25d3 100644 --- a/inttest/erlc/erlc_rt.erl +++ b/inttest/erlc/erlc_rt.erl @@ -66,6 +66,10 @@ files() -> {copy, "mibs", "mibs"}, {copy, "asn1", "asn1"}, {create, "ebin/foo.app", app(foo, ?MODULES)}, + {create, "src/._do_not_compile.erl", + "syntax error\n" + "this file is here to verify that rebar does not try to compile\n" + "files like OS X resource forks and should not be processed at all\n"}, %% deps {create, "deps/foobar/ebin/foobar.app", app(foobar, [foobar])}, {copy, "foobar.erl", "deps/foobar/src/foobar.erl"} diff --git a/inttest/erlc/src/._do_not_compile.erl b/inttest/erlc/src/._do_not_compile.erl deleted file mode 100644 index f26f9879..00000000 --- a/inttest/erlc/src/._do_not_compile.erl +++ /dev/null @@ -1,4 +0,0 @@ -syntax error -this file is here to verify that rebar does not try to -compile files like OS X resource forks and should not be -processed at all From 24d82c3befff262b62e6fb064b973c90873f642d Mon Sep 17 00:00:00 2001 From: Tony Rogvall Date: Tue, 20 Jan 2015 21:54:22 +0100 Subject: [PATCH 03/89] port_compiler: add cross arch support New vars for cross arch support: * REBAR_ARCH_TARGET * REBAR_TARGET_WORDSIZE * REBAR_ARCH_TARGET_VSN --- src/rebar_port_compiler.erl | 29 +++++++++++++++++-- src/rebar_utils.erl | 58 ++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 58555896..1c24b2fb 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -568,9 +568,21 @@ erl_interface_dir(Subdir) -> end. default_env() -> + Arch = os:getenv("REBAR_ARCH_TARGET"), + Vsn = os:getenv("REBAR_ARCH_TARGET_VSN"), [ - {"CC" , "cc"}, - {"CXX", "c++"}, + {"CC" , get_tool(Arch,Vsn,"gcc","cc")}, + {"CXX", get_tool(Arch,Vsn,"g++","c++")}, + {"AR" , get_tool(Arch,"ar","ar")}, + {"AS" , get_tool(Arch,"as","as")}, + {"CPP" , get_tool(Arch,Vsn,"cpp","cpp")}, + {"LD" , get_tool(Arch,"ld","ld")}, + {"RANLIB" , get_tool(Arch,Vsn,"ranlib","ranlib")}, + {"STRIP" , get_tool(Arch,"strip","strip")}, + {"NM" , get_tool(Arch,"nm","nm")}, + {"OBJCOPY" , get_tool(Arch,"objcopy","objcopy")}, + {"OBJDUMP" , get_tool(Arch,"objdump","objdump")}, + {"DRV_CXX_TEMPLATE", "$CXX -c $CXXFLAGS $DRV_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, {"DRV_CC_TEMPLATE", @@ -643,3 +655,16 @@ default_env() -> {"win32", "DRV_CFLAGS", "/Zi /Wall $ERL_CFLAGS"}, {"win32", "DRV_LDFLAGS", "/DLL $ERL_LDFLAGS"} ]. + +get_tool(Arch,Tool,Default) -> + get_tool(Arch,false,Tool,Default). + +get_tool(false, _, _, Default) -> Default; +get_tool("",_,_, Default) -> Default; +get_tool(Arch, false, Tool, _Default) -> Arch++"-"++Tool; +get_tool(Arch, "", Tool, _Default) -> Arch++"-"++Tool; +get_tool(Arch, Vsn, Tool, _Default) -> Arch++"-"++Tool++"-"++Vsn. + + + + diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 96817561..75798abf 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -91,13 +91,63 @@ is_arch(ArchRegex) -> nomatch -> false end. - +%% +%% REBAR_ARCH_TARGET, if used, should be set to the "standard" +%% target string. That is a prefix for binutils tools. +%% "x86_64-linux-gnu" or "arm-linux-gnueabi" are good candiates +%% ${REBAR_ARCH_TARGET}-gcc, ${REBAR_ARCH_TARGET}-ld ... +%% get_arch() -> - Words = wordsize(), - otp_release() ++ "-" - ++ erlang:system_info(system_architecture) ++ "-" ++ Words. + Arch = os:getenv("REBAR_ARCH_TARGET"), + Words = wordsize(Arch), + otp_release() ++ "-" ++ get_system_arch(Arch) ++ "-" ++ Words. + +get_system_arch(Arch) when Arch =:= false; Arch =:= "" -> + erlang:system_info(system_architecture); +get_system_arch(Arch) -> + Arch. wordsize() -> + wordsize(os:getenv("REBAR_ARCH_TARGET")). + +wordsize(Arch) when Arch =:= false; Arch =:= "" -> + native_wordsize(); +wordsize(Arch) -> + %% add a simple call to gcc to find out? + case match_wordsize(Arch, [{"i686","32"}, {"i386","32"}, + {"arm","32"}, {"x86_64","64"}]) of + false -> + env_wordsize(os:getenv("REBAR_ARCH_WORDSIZE")); + {_, Wordsize} -> + Wordsize + end. + +match_wordsize(Arch, [V={Match,_Bits}|Vs]) -> + case re:run(Arch, Match, [{capture, none}]) of + match -> V; + _ -> + match_wordsize(Arch, Vs) + end. + +env_wordsize(Wordsize) when Wordsize =:= false; + Wordsize =:= "" -> + io:format("REBAR_ARCH_WORDSIZE not set, assuming 32\n"), + "32"; +env_wordsize(Wordsize) -> + try list_to_integer(Wordsize) of + 16 -> "16"; + 32 -> "32"; + 64 -> "64"; + _ -> + io:format("REBAR_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), + "32" + catch + error:_ -> + io:format("REBAR_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), + "32" + end. + +native_wordsize() -> try erlang:system_info({wordsize, external}) of Val -> integer_to_list(8 * Val) From 29447c760f388d0b5d288e10846129ace6e66f20 Mon Sep 17 00:00:00 2001 From: Tony Rogvall Date: Mon, 26 Jan 2015 16:10:35 +0100 Subject: [PATCH 04/89] rebar_utils: add (word)size helper function --- src/rebar_utils.erl | 76 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 75798abf..73218455 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -67,7 +67,13 @@ processing_base_dir/1, processing_base_dir/2, patch_env/2, - cleanup_code_path/1]). + cleanup_code_path/1, + cross_wordsize/1, + native_wordsize/0, + wordsize/1, + cross_sizeof/2, + env_wordsize/1 + ]). %% for internal use only -export([otp_release/0]). @@ -113,11 +119,14 @@ wordsize() -> wordsize(Arch) when Arch =:= false; Arch =:= "" -> native_wordsize(); wordsize(Arch) -> - %% add a simple call to gcc to find out? case match_wordsize(Arch, [{"i686","32"}, {"i386","32"}, {"arm","32"}, {"x86_64","64"}]) of false -> - env_wordsize(os:getenv("REBAR_ARCH_WORDSIZE")); + case cross_wordsize(Arch) of + "" -> + env_wordsize(os:getenv("REBAR_ARCH_WORDSIZE")); + WordSize -> WordSize + end; {_, Wordsize} -> Wordsize end. @@ -147,6 +156,67 @@ env_wordsize(Wordsize) -> "32" end. +%% +%% Findout the word size of the target by using Arch-gcc +%% + +cross_wordsize(Arch) -> + cross_sizeof(Arch, "void*"). + +%% +%% Find the size of target Type using a specially crafted C file +%% that will report an error on the line of the byte size of the type. +%% + +cross_sizeof(Arch, Type) -> + Compiler = if Arch =:= "" -> "cc"; + true -> Arch ++ "-gcc" + end, + TempFile = mktemp(".c"), + file:write_file(TempFile, + <<"int t01 [1 - 2*(((long) (sizeof (TYPE))) == 1)];\n" + "int t02 [1 - 2*(((long) (sizeof (TYPE))) == 2)];\n" + "int t03 [1 - 2*(((long) (sizeof (TYPE))) == 3)];\n" + "int t04 [1 - 2*(((long) (sizeof (TYPE))) == 4)];\n" + "int t05 [1 - 2*(((long) (sizeof (TYPE))) == 5)];\n" + "int t06 [1 - 2*(((long) (sizeof (TYPE))) == 6)];\n" + "int t07 [1 - 2*(((long) (sizeof (TYPE))) == 7)];\n" + "int t08 [1 - 2*(((long) (sizeof (TYPE))) == 8)];\n" + "int t09 [1 - 2*(((long) (sizeof (TYPE))) == 9)];\n" + "int t10 [1 - 2*(((long) (sizeof (TYPE))) == 10)];\n" + "int t11 [1 - 2*(((long) (sizeof (TYPE))) == 11)];\n" + "int t12 [1 - 2*(((long) (sizeof (TYPE))) == 12)];\n" + "int t13 [1 - 2*(((long) (sizeof (TYPE))) == 13)];\n" + "int t14 [1 - 2*(((long) (sizeof (TYPE))) == 14)];\n" + "int t15 [1 - 2*(((long) (sizeof (TYPE))) == 15)];\n" + "int t16 [1 - 2*(((long) (sizeof (TYPE))) == 16)];\n" + >>), + Res = os:cmd(Compiler ++ " -DTYPE=\""++Type++"\" " ++ TempFile), + file:delete(TempFile), + case string:tokens(Res, ":") of + [_,Ln | _] -> + try list_to_integer(Ln) of + NumBytes -> integer_to_list(NumBytes*8) + catch + error:_ -> + "" + end; + _ -> + "" + end. + +mktemp(Suffix) -> + {A,B,C} = erlang:now(), + Dir = case os:type() of + {windows,_} -> "C:/WINDOWS/TEMP"; + _ -> "/tmp" + end, + File = "rebar_"++os:getpid()++ + integer_to_list(A)++"_"++ + integer_to_list(B)++"_"++ + integer_to_list(C)++Suffix, + filename:join(Dir,File). + native_wordsize() -> try erlang:system_info({wordsize, external}) of Val -> From 49fd99c7011191ae9fe499f03c8070faf0bb41ba Mon Sep 17 00:00:00 2001 From: Tony Rogvall Date: Thu, 5 Feb 2015 15:11:14 +0100 Subject: [PATCH 05/89] port_compiler: cross-arch rename variables New variable names: * REBAR_TARGET_ARCH * REBAR_TARGET_ARCH_VSN * REBAR_TARGET_ARCH_WORDSIZE --- src/rebar_port_compiler.erl | 4 ++-- src/rebar_utils.erl | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 1c24b2fb..84cf49ba 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -568,8 +568,8 @@ erl_interface_dir(Subdir) -> end. default_env() -> - Arch = os:getenv("REBAR_ARCH_TARGET"), - Vsn = os:getenv("REBAR_ARCH_TARGET_VSN"), + Arch = os:getenv("REBAR_TARGET_ARCH"), + Vsn = os:getenv("REBAR_TARGET_ARCH_VSN"), [ {"CC" , get_tool(Arch,Vsn,"gcc","cc")}, {"CXX", get_tool(Arch,Vsn,"g++","c++")}, diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 73218455..31ccb3f7 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -98,13 +98,13 @@ is_arch(ArchRegex) -> false end. %% -%% REBAR_ARCH_TARGET, if used, should be set to the "standard" +%% REBAR_TARGET_ARCH, if used, should be set to the "standard" %% target string. That is a prefix for binutils tools. %% "x86_64-linux-gnu" or "arm-linux-gnueabi" are good candiates -%% ${REBAR_ARCH_TARGET}-gcc, ${REBAR_ARCH_TARGET}-ld ... +%% ${REBAR_TARGET_ARCH}-gcc, ${REBAR_TARGET_ARCH}-ld ... %% get_arch() -> - Arch = os:getenv("REBAR_ARCH_TARGET"), + Arch = os:getenv("REBAR_TARGET_ARCH"), Words = wordsize(Arch), otp_release() ++ "-" ++ get_system_arch(Arch) ++ "-" ++ Words. @@ -114,17 +114,18 @@ get_system_arch(Arch) -> Arch. wordsize() -> - wordsize(os:getenv("REBAR_ARCH_TARGET")). + wordsize(os:getenv("REBAR_TARGET_ARCH")). wordsize(Arch) when Arch =:= false; Arch =:= "" -> native_wordsize(); wordsize(Arch) -> case match_wordsize(Arch, [{"i686","32"}, {"i386","32"}, - {"arm","32"}, {"x86_64","64"}]) of + {"arm","32"}, {"aarch64", "64"}, + {"x86_64","64"}]) of false -> case cross_wordsize(Arch) of "" -> - env_wordsize(os:getenv("REBAR_ARCH_WORDSIZE")); + env_wordsize(os:getenv("REBAR_TARGET_ARCH_WORDSIZE")); WordSize -> WordSize end; {_, Wordsize} -> @@ -140,7 +141,7 @@ match_wordsize(Arch, [V={Match,_Bits}|Vs]) -> env_wordsize(Wordsize) when Wordsize =:= false; Wordsize =:= "" -> - io:format("REBAR_ARCH_WORDSIZE not set, assuming 32\n"), + io:format("REBAR_TARGET_ARCH_WORDSIZE not set, assuming 32\n"), "32"; env_wordsize(Wordsize) -> try list_to_integer(Wordsize) of @@ -148,11 +149,11 @@ env_wordsize(Wordsize) -> 32 -> "32"; 64 -> "64"; _ -> - io:format("REBAR_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), + io:format("REBAR_TARGET_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), "32" catch error:_ -> - io:format("REBAR_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), + io:format("REBAR_TARGET_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), "32" end. From 9a64826172ade12997e8ee0ff32df52757c43025 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Thu, 2 Jul 2015 13:45:35 +0200 Subject: [PATCH 06/89] cross-arch: fix up issues * fix commit messages * fix whitespace issues * move internal helper functions to correct location * unexport internal cross arch helper functions * fix 18.0 time API compatibility * fix Windows temp dir detection * use correct helper function name * use sh/2 instead of os:cmd/1 * match file:delete/1 result * use logging macros * fix typo * fix Dialyzer warnings * add Tony Rogvall to THANKS * add missing termination clause * os type must be win32, not windows * match file:write_file/2 result * document cross-arch variables * simplify env_wordsize/1 (Thanks Fred Hebert) --- THANKS | 1 + rebar.config | 3 +- src/rebar_port_compiler.erl | 59 ++++---- src/rebar_utils.erl | 265 ++++++++++++++++++++---------------- 4 files changed, 183 insertions(+), 145 deletions(-) diff --git a/THANKS b/THANKS index 042bb303..2ba6abe2 100644 --- a/THANKS +++ b/THANKS @@ -142,3 +142,4 @@ Derek Brown Danil Onishchenko Stavros Aronis James Fish +Tony Rogvall diff --git a/rebar.config b/rebar.config index a3849ec6..bcd53685 100644 --- a/rebar.config +++ b/rebar.config @@ -29,7 +29,8 @@ - (\"gpb_compile\":\"format_error\"/\"1\") - (\"diameter_codegen\":\"from_dict\"/\"4\") - (\"diameter_dict_util\":\"format_error\"/\"1\") - - (\"diameter_dict_util\":\"parse\"/\"2\"))", + - (\"diameter_dict_util\":\"parse\"/\"2\") + - (\"erlang\":\"timestamp\"/\"0\"))", []}]}. {dialyzer, diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 84cf49ba..dcd8a20a 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -165,7 +165,14 @@ info_help(Description) -> "~n" "Valid rebar.config options:~n" " ~p~n" - " ~p~n", + " ~p~n" + "Cross-arch environment variables:~n" + " REBAR_TARGET_ARCH to set the tool chain name to use~n" + " REBAR_TARGET_ARCH_WORDSIZE optional " + "(if CC fails to determine word size)~n" + " fallback word size is 32~n" + " REBAR_TARGET_ARCH_VSN optional " + "(if a special version of CC/CXX is requested)~n", [ Description, {port_env, [{"CFLAGS", "$CFLAGS -Ifoo"}, @@ -183,8 +190,10 @@ setup_env(Config, ExtraEnv) -> %% Get any port-specific envs; use port_env first and then fallback %% to port_envs for compatibility - RawPortEnv = rebar_config:get_list(Config, port_env, - rebar_config:get_list(Config, port_envs, [])), + RawPortEnv = rebar_config:get_list( + Config, + port_env, + rebar_config:get_list(Config, port_envs, [])), PortEnv = filter_env(RawPortEnv, []), Defines = get_defines(Config), @@ -571,17 +580,17 @@ default_env() -> Arch = os:getenv("REBAR_TARGET_ARCH"), Vsn = os:getenv("REBAR_TARGET_ARCH_VSN"), [ - {"CC" , get_tool(Arch,Vsn,"gcc","cc")}, - {"CXX", get_tool(Arch,Vsn,"g++","c++")}, - {"AR" , get_tool(Arch,"ar","ar")}, - {"AS" , get_tool(Arch,"as","as")}, - {"CPP" , get_tool(Arch,Vsn,"cpp","cpp")}, - {"LD" , get_tool(Arch,"ld","ld")}, - {"RANLIB" , get_tool(Arch,Vsn,"ranlib","ranlib")}, - {"STRIP" , get_tool(Arch,"strip","strip")}, - {"NM" , get_tool(Arch,"nm","nm")}, - {"OBJCOPY" , get_tool(Arch,"objcopy","objcopy")}, - {"OBJDUMP" , get_tool(Arch,"objdump","objdump")}, + {"CC", get_tool(Arch, Vsn,"gcc", "cc")}, + {"CXX", get_tool(Arch, Vsn,"g++", "c++")}, + {"AR", get_tool(Arch, "ar", "ar")}, + {"AS", get_tool(Arch, "as", "as")}, + {"CPP", get_tool(Arch, Vsn, "cpp", "cpp")}, + {"LD", get_tool(Arch, "ld", "ld")}, + {"RANLIB", get_tool(Arch, Vsn, "ranlib", "ranlib")}, + {"STRIP", get_tool(Arch, "strip", "strip")}, + {"NM", get_tool(Arch, "nm", "nm")}, + {"OBJCOPY", get_tool(Arch, "objcopy", "objcopy")}, + {"OBJDUMP", get_tool(Arch, "objdump", "objdump")}, {"DRV_CXX_TEMPLATE", "$CXX -c $CXXFLAGS $DRV_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, @@ -600,9 +609,12 @@ default_env() -> {"EXE_CFLAGS" , "-g -Wall -fPIC -MMD $ERL_CFLAGS"}, {"EXE_LDFLAGS", "$ERL_LDFLAGS"}, - {"ERL_CFLAGS", lists:concat([" -I\"", erl_interface_dir(include), - "\" -I\"", filename:join(erts_dir(), "include"), - "\" "])}, + {"ERL_CFLAGS", lists:concat( + [ + " -I\"", erl_interface_dir(include), + "\" -I\"", filename:join(erts_dir(), "include"), + "\" " + ])}, {"ERL_EI_LIBDIR", lists:concat(["\"", erl_interface_dir(lib), "\""])}, {"ERL_LDFLAGS" , " -L$ERL_EI_LIBDIR -lerl_interface -lei"}, {"ERLANG_ARCH" , rebar_utils:wordsize()}, @@ -651,20 +663,17 @@ default_env() -> {"win32", "EXE_LINK_TEMPLATE", "$LINKER $PORT_IN_FILES $LDFLAGS $EXE_LDFLAGS /OUT:$PORT_OUT_FILE"}, %% ERL_CFLAGS are ok as -I even though strictly it should be /I - {"win32", "ERL_LDFLAGS", " /LIBPATH:$ERL_EI_LIBDIR erl_interface.lib ei.lib"}, + {"win32", "ERL_LDFLAGS", + " /LIBPATH:$ERL_EI_LIBDIR erl_interface.lib ei.lib"}, {"win32", "DRV_CFLAGS", "/Zi /Wall $ERL_CFLAGS"}, {"win32", "DRV_LDFLAGS", "/DLL $ERL_LDFLAGS"} ]. -get_tool(Arch,Tool,Default) -> - get_tool(Arch,false,Tool,Default). +get_tool(Arch, Tool, Default) -> + get_tool(Arch, false, Tool, Default). get_tool(false, _, _, Default) -> Default; -get_tool("",_,_, Default) -> Default; +get_tool("", _, _, Default) -> Default; get_tool(Arch, false, Tool, _Default) -> Arch++"-"++Tool; get_tool(Arch, "", Tool, _Default) -> Arch++"-"++Tool; get_tool(Arch, Vsn, Tool, _Default) -> Arch++"-"++Tool++"-"++Vsn. - - - - diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 31ccb3f7..045e67b2 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -67,12 +67,7 @@ processing_base_dir/1, processing_base_dir/2, patch_env/2, - cleanup_code_path/1, - cross_wordsize/1, - native_wordsize/0, - wordsize/1, - cross_sizeof/2, - env_wordsize/1 + cleanup_code_path/1 ]). %% for internal use only @@ -98,9 +93,9 @@ is_arch(ArchRegex) -> false end. %% -%% REBAR_TARGET_ARCH, if used, should be set to the "standard" +%% REBAR_TARGET_ARCH, if used, should be set to the "standard" %% target string. That is a prefix for binutils tools. -%% "x86_64-linux-gnu" or "arm-linux-gnueabi" are good candiates +%% "x86_64-linux-gnu" or "arm-linux-gnueabi" are good candidates %% ${REBAR_TARGET_ARCH}-gcc, ${REBAR_TARGET_ARCH}-ld ... %% get_arch() -> @@ -116,117 +111,6 @@ get_system_arch(Arch) -> wordsize() -> wordsize(os:getenv("REBAR_TARGET_ARCH")). -wordsize(Arch) when Arch =:= false; Arch =:= "" -> - native_wordsize(); -wordsize(Arch) -> - case match_wordsize(Arch, [{"i686","32"}, {"i386","32"}, - {"arm","32"}, {"aarch64", "64"}, - {"x86_64","64"}]) of - false -> - case cross_wordsize(Arch) of - "" -> - env_wordsize(os:getenv("REBAR_TARGET_ARCH_WORDSIZE")); - WordSize -> WordSize - end; - {_, Wordsize} -> - Wordsize - end. - -match_wordsize(Arch, [V={Match,_Bits}|Vs]) -> - case re:run(Arch, Match, [{capture, none}]) of - match -> V; - _ -> - match_wordsize(Arch, Vs) - end. - -env_wordsize(Wordsize) when Wordsize =:= false; - Wordsize =:= "" -> - io:format("REBAR_TARGET_ARCH_WORDSIZE not set, assuming 32\n"), - "32"; -env_wordsize(Wordsize) -> - try list_to_integer(Wordsize) of - 16 -> "16"; - 32 -> "32"; - 64 -> "64"; - _ -> - io:format("REBAR_TARGET_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), - "32" - catch - error:_ -> - io:format("REBAR_TARGET_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), - "32" - end. - -%% -%% Findout the word size of the target by using Arch-gcc -%% - -cross_wordsize(Arch) -> - cross_sizeof(Arch, "void*"). - -%% -%% Find the size of target Type using a specially crafted C file -%% that will report an error on the line of the byte size of the type. -%% - -cross_sizeof(Arch, Type) -> - Compiler = if Arch =:= "" -> "cc"; - true -> Arch ++ "-gcc" - end, - TempFile = mktemp(".c"), - file:write_file(TempFile, - <<"int t01 [1 - 2*(((long) (sizeof (TYPE))) == 1)];\n" - "int t02 [1 - 2*(((long) (sizeof (TYPE))) == 2)];\n" - "int t03 [1 - 2*(((long) (sizeof (TYPE))) == 3)];\n" - "int t04 [1 - 2*(((long) (sizeof (TYPE))) == 4)];\n" - "int t05 [1 - 2*(((long) (sizeof (TYPE))) == 5)];\n" - "int t06 [1 - 2*(((long) (sizeof (TYPE))) == 6)];\n" - "int t07 [1 - 2*(((long) (sizeof (TYPE))) == 7)];\n" - "int t08 [1 - 2*(((long) (sizeof (TYPE))) == 8)];\n" - "int t09 [1 - 2*(((long) (sizeof (TYPE))) == 9)];\n" - "int t10 [1 - 2*(((long) (sizeof (TYPE))) == 10)];\n" - "int t11 [1 - 2*(((long) (sizeof (TYPE))) == 11)];\n" - "int t12 [1 - 2*(((long) (sizeof (TYPE))) == 12)];\n" - "int t13 [1 - 2*(((long) (sizeof (TYPE))) == 13)];\n" - "int t14 [1 - 2*(((long) (sizeof (TYPE))) == 14)];\n" - "int t15 [1 - 2*(((long) (sizeof (TYPE))) == 15)];\n" - "int t16 [1 - 2*(((long) (sizeof (TYPE))) == 16)];\n" - >>), - Res = os:cmd(Compiler ++ " -DTYPE=\""++Type++"\" " ++ TempFile), - file:delete(TempFile), - case string:tokens(Res, ":") of - [_,Ln | _] -> - try list_to_integer(Ln) of - NumBytes -> integer_to_list(NumBytes*8) - catch - error:_ -> - "" - end; - _ -> - "" - end. - -mktemp(Suffix) -> - {A,B,C} = erlang:now(), - Dir = case os:type() of - {windows,_} -> "C:/WINDOWS/TEMP"; - _ -> "/tmp" - end, - File = "rebar_"++os:getpid()++ - integer_to_list(A)++"_"++ - integer_to_list(B)++"_"++ - integer_to_list(C)++Suffix, - filename:join(Dir,File). - -native_wordsize() -> - try erlang:system_info({wordsize, external}) of - Val -> - integer_to_list(8 * Val) - catch - error:badarg -> - integer_to_list(8 * erlang:system_info(wordsize)) - end. - sh_send(Command0, String, Options0) -> ?INFO("sh_send info:\n\tcwd: ~p\n\tcmd: ~s < ~s\n", [get_cwd(), Command0, String]), @@ -794,3 +678,146 @@ cleanup_code_path(OrigPath) -> _ -> code:set_path(OrigPath) end. + +wordsize(Arch) when Arch =:= false; Arch =:= "" -> + native_wordsize(); +wordsize(Arch) -> + AllArchs = [ + {"i686","32"}, + {"i386","32"}, + {"arm","32"}, + {"aarch64", "64"}, + {"x86_64","64"} + ], + case match_wordsize(Arch, AllArchs) of + false -> + case cross_wordsize(Arch) of + "" -> + env_wordsize(os:getenv("REBAR_TARGET_ARCH_WORDSIZE")); + WordSize -> + WordSize + end; + {_, Wordsize} -> + Wordsize + end. + +match_wordsize(Arch, [V={Match,_Bits}|Vs]) -> + case re:run(Arch, Match, [{capture, none}]) of + match -> + V; + nomatch -> + match_wordsize(Arch, Vs) + end; +match_wordsize(_Arch, []) -> + false. + +env_wordsize(Wordsize) when Wordsize =:= false; + Wordsize =:= "" -> + ?WARN("REBAR_TARGET_ARCH_WORDSIZE not set, assuming 32\n", []), + "32"; +env_wordsize(Wordsize) -> + case Wordsize of + "16" -> Wordsize; + "32" -> Wordsize; + "64" -> Wordsize; + _ -> + ?WARN("REBAR_TARGET_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), + "32" + end. + +%% +%% Find out the word size of the target by using Arch-gcc +%% +cross_wordsize(Arch) -> + cross_sizeof(Arch, "void*"). + +%% +%% Find the size of target Type using a specially crafted C file +%% that will report an error on the line of the byte size of the type. +%% +cross_sizeof(Arch, Type) -> + Compiler = if Arch =:= "" -> "cc"; + true -> Arch ++ "-gcc" + end, + TempFile = mktempfile(".c"), + ok = file:write_file(TempFile, + <<"int t01 [1 - 2*(((long) (sizeof (TYPE))) == 1)];\n" + "int t02 [1 - 2*(((long) (sizeof (TYPE))) == 2)];\n" + "int t03 [1 - 2*(((long) (sizeof (TYPE))) == 3)];\n" + "int t04 [1 - 2*(((long) (sizeof (TYPE))) == 4)];\n" + "int t05 [1 - 2*(((long) (sizeof (TYPE))) == 5)];\n" + "int t06 [1 - 2*(((long) (sizeof (TYPE))) == 6)];\n" + "int t07 [1 - 2*(((long) (sizeof (TYPE))) == 7)];\n" + "int t08 [1 - 2*(((long) (sizeof (TYPE))) == 8)];\n" + "int t09 [1 - 2*(((long) (sizeof (TYPE))) == 9)];\n" + "int t10 [1 - 2*(((long) (sizeof (TYPE))) == 10)];\n" + "int t11 [1 - 2*(((long) (sizeof (TYPE))) == 11)];\n" + "int t12 [1 - 2*(((long) (sizeof (TYPE))) == 12)];\n" + "int t13 [1 - 2*(((long) (sizeof (TYPE))) == 13)];\n" + "int t14 [1 - 2*(((long) (sizeof (TYPE))) == 14)];\n" + "int t15 [1 - 2*(((long) (sizeof (TYPE))) == 15)];\n" + "int t16 [1 - 2*(((long) (sizeof (TYPE))) == 16)];\n" + >>), + Cmd = Compiler ++ " -DTYPE=\""++Type++"\" " ++ TempFile, + ShOpts = [{use_stdout, false}, return_on_error], + {ok, Res} = sh(Cmd, ShOpts), + ok = file:delete(TempFile), + case string:tokens(Res, ":") of + [_, Ln | _] -> + try list_to_integer(Ln) of + NumBytes -> integer_to_list(NumBytes*8) + catch + error:_ -> + "" + end; + _ -> + "" + end. + +mktempfile(Suffix) -> + {A,B,C} = rebar_now(), + Dir = temp_dir(), + File = "rebar_"++os:getpid()++ + integer_to_list(A)++"_"++ + integer_to_list(B)++"_"++ + integer_to_list(C)++Suffix, + filename:join(Dir, File). + +temp_dir() -> + case os:type() of + {win32, _} -> windows_temp_dir(); + _ -> "/tmp" + end. + +windows_temp_dir() -> + case os:getenv("TEMP") of + false -> + case os:getenv("TMP") of + false -> "C:/WINDOWS/TEMP"; + TMP -> TMP + end; + TEMP -> TEMP + end. + +rebar_now() -> + case erlang:function_exported(erlang, timestamp, 0) of + true -> + erlang:timestamp(); + false -> + %% erlang:now/0 was deprecated in 18.0, and as the escript has to + %% pass erl_lint:module/1 (even without -mode(compile)), we would + %% see a deprecation warning for erlang:now/0. One solution is to + %% use -compile({nowarn_deprecated_function, [{erlang, now, 0}]}), + %% but that would raise a warning in versions older than 18.0. + %% Calling erlang:now/0 via apply/3 avoids that. + apply(erlang, now, []) + end. + +native_wordsize() -> + try erlang:system_info({wordsize, external}) of + Val -> + integer_to_list(8 * Val) + catch + error:badarg -> + integer_to_list(8 * erlang:system_info(wordsize)) + end. From 4780cc4e3b94624dab5fc8015bb683c2b41f0989 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Thu, 2 Jul 2015 21:24:16 +0200 Subject: [PATCH 07/89] rebar_utils: fix comment --- src/rebar_utils.erl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 045e67b2..64595a2f 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -804,12 +804,11 @@ rebar_now() -> true -> erlang:timestamp(); false -> - %% erlang:now/0 was deprecated in 18.0, and as the escript has to - %% pass erl_lint:module/1 (even without -mode(compile)), we would - %% see a deprecation warning for erlang:now/0. One solution is to - %% use -compile({nowarn_deprecated_function, [{erlang, now, 0}]}), - %% but that would raise a warning in versions older than 18.0. - %% Calling erlang:now/0 via apply/3 avoids that. + %% erlang:now/0 was deprecated in 18.0. One solution to avoid the + %% deprecation warning is to use + %% -compile({nowarn_deprecated_function, [{erlang, now, 0}]}), but + %% that would raise a warning in versions older than 18.0. Calling + %% erlang:now/0 via apply/3 avoids that. apply(erlang, now, []) end. From a587a9f7eb10691b7da5fcb45f2741c82618142f Mon Sep 17 00:00:00 2001 From: Andrey Teplyashin Date: Sun, 9 Nov 2014 14:04:05 +0600 Subject: [PATCH 08/89] Add colored logs Fix verbose level in global rebar config Replace application:set_env on erlang:put --- THANKS | 1 + ebin/rebar.app | 3 ++ inttest/logging/logging_rt.erl | 31 ++++++++++--- .../t_custom_config/t_custom_config_rt.erl | 5 ++- inttest/tplugins/tplugins_rt.erl | 2 +- src/rebar_log.erl | 45 +++++++++++++++++-- 6 files changed, 73 insertions(+), 14 deletions(-) diff --git a/THANKS b/THANKS index 2ba6abe2..7a5f5ecd 100644 --- a/THANKS +++ b/THANKS @@ -143,3 +143,4 @@ Danil Onishchenko Stavros Aronis James Fish Tony Rogvall +Andrey Teplyashin diff --git a/ebin/rebar.app b/ebin/rebar.app index 152829da..985dbb96 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -68,6 +68,9 @@ %% Default log level {log_level, warn}, + %% Log colored + {log_colored, true}, + %% any_dir processing modules {any_dir_modules, [ rebar_require_vsn, diff --git a/inttest/logging/logging_rt.erl b/inttest/logging/logging_rt.erl index d3e1c0fd..11cb0cdb 100644 --- a/inttest/logging/logging_rt.erl +++ b/inttest/logging/logging_rt.erl @@ -40,23 +40,40 @@ run(_Dir) -> SharedExpected = "==> logging_rt \\(compile\\)", %% provoke ERROR due to an invalid app file retest:log(info, "Check 'compile' failure output~n"), + {ERROR, WARN, INFO, DEBUG} = + case application:get_env(rebar, log_colored) of + {ok, true} -> + { + "\\e\\[1m\\e\\[31mERROR: \\e\\[0m", + "\\e\\[33mWARN: \\e\\[0m", + "\\e\\[32mINFO: \\e\\[0m", + "\\e\\[34mDEBUG: \\e\\[0m" + }; + _ -> + { + "ERROR: ", + "WARN: ", + "INFO: ", + "DEBUG: " + } + end, ok = check_output("./rebar compile -q", should_fail, - [SharedExpected, "ERROR: "], - ["WARN: ", "INFO: ", "DEBUG: "]), + [SharedExpected, ERROR], + [WARN, INFO, DEBUG]), %% fix bad app file ok = file:write_file(?APP_FILE, app(logging, [])), retest:log(info, "Check 'compile' success output~n"), ok = check_output("./rebar compile", should_succeed, [SharedExpected], - ["ERROR: ", "WARN: ", "INFO: ", "DEBUG: "]), + [ERROR, WARN, INFO, DEBUG]), retest:log(info, "Check 'compile -v' success output~n"), ok = check_output("./rebar compile -v", should_succeed, [SharedExpected], - ["ERROR: ", "INFO: ", "DEBUG: "]), + [ERROR, INFO, DEBUG]), retest:log(info, "Check 'compile -vv' success output~n"), ok = check_output("./rebar compile -vv", should_succeed, - [SharedExpected, "DEBUG: "], - ["ERROR: ", "INFO: "]), + [SharedExpected, DEBUG], + [ERROR, INFO]), ok. check_output(Cmd, FailureMode, Expected, Unexpected) -> @@ -73,7 +90,7 @@ check_output(Cmd, FailureMode, Expected, Unexpected) -> end. check_output1(Cmd, Captured, Expected, Unexpected) -> - ReOpts = [{capture, all, list}], + ReOpts = [{capture, all, list}, unicode], ExMatches = lists:zf( fun(Pattern) -> diff --git a/inttest/t_custom_config/t_custom_config_rt.erl b/inttest/t_custom_config/t_custom_config_rt.erl index a6aac9b4..a373b327 100644 --- a/inttest/t_custom_config/t_custom_config_rt.erl +++ b/inttest/t_custom_config/t_custom_config_rt.erl @@ -15,13 +15,14 @@ run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), Ref = retest:sh("./rebar -C custom.config check-deps -vv", [{async, true}]), + {ok, Captured} = retest:sh_expect(Ref, - "DEBUG: Consult config file .*/custom.config.*", + ".*DEBUG: .*Consult config file .*/custom.config.*", [{capture, all, list}]), {ok, Missing} = retest:sh_expect(Ref, - "DEBUG: Missing deps : \\[\\{dep,bad_name," + ".*DEBUG: .*Missing deps : \\[\\{dep,bad_name," "boo,\"\\.\",undefined,false\\}\\]", [{capture, all, list}]), retest_log:log(debug, "[CAPTURED]: ~s~n", [Captured]), diff --git a/inttest/tplugins/tplugins_rt.erl b/inttest/tplugins/tplugins_rt.erl index 997661b4..01d296ea 100644 --- a/inttest/tplugins/tplugins_rt.erl +++ b/inttest/tplugins/tplugins_rt.erl @@ -24,7 +24,7 @@ run(_Dir) -> ?assertMatch({ok, _}, retest_sh:run("./rebar fwibble -v", [])), ?assertEqual(false, filelib:is_regular("fwibble.test")), Ref = retest:sh("./rebar -C bad.config -v clean", [{async, true}]), - {ok, _} = retest:sh_expect(Ref, "ERROR: Plugin .*bad_plugin.erl " + {ok, _} = retest:sh_expect(Ref, ".*ERROR: .*Plugin .*bad_plugin.erl " "contains compilation errors:.*", [{newline, any}]), ok. diff --git a/src/rebar_log.erl b/src/rebar_log.erl index ba25332c..91fb6596 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -50,19 +50,30 @@ init(Config) -> ?WARN_LEVEL -> set_level(warn); ?INFO_LEVEL -> set_level(info); ?DEBUG_LEVEL -> set_level(debug) - end. + end, + LogColored = rebar_config:get_global(Config, log_colored, true), + set_log_colored(LogColored). + set_level(Level) -> - ok = application:set_env(rebar, log_level, Level). + erlang:put(rebar_log_level, Level). + +set_log_colored(true) -> + erlang:put(rebar_log_colored, true), + ok; +set_log_colored(_LogColored) -> + erlang:put(rebar_log_colored, false), + ok. log(Level, Str, Args) -> log(standard_io, Level, Str, Args). log(Device, Level, Str, Args) -> - {ok, LogLevel} = application:get_env(rebar, log_level), + LogLevel = erlang:get(rebar_log_level), + LogColored = erlang:get(rebar_log_colored), case should_log(LogLevel, Level) of true -> - io:format(Device, log_prefix(Level) ++ Str, Args); + io:format(Device, log_prefix(Level, LogColored) ++ Str, Args); false -> ok end. @@ -90,7 +101,33 @@ should_log(error, error) -> true; should_log(error, _) -> false; should_log(_, _) -> false. +log_prefix(Level, _Colored = false) -> + log_prefix(Level); +log_prefix(Level, _Colored = true) -> + color_from_level(Level) ++ log_prefix(Level) ++ reset_color(). + log_prefix(debug) -> "DEBUG: "; log_prefix(info) -> "INFO: "; log_prefix(warn) -> "WARN: "; log_prefix(error) -> "ERROR: ". + +color_from_level(debug) -> + color_foreground(blue); +color_from_level(info) -> + color_foreground(green); +color_from_level(warn) -> + color_foreground(yellow); +color_from_level(error) -> + color_bold() ++ color_foreground(red). + +color_foreground(black) -> "\e[30m"; +color_foreground(red) -> "\e[31m"; +color_foreground(green) -> "\e[32m"; +color_foreground(yellow) -> "\e[33m"; +color_foreground(blue) -> "\e[34m"; +color_foreground(magenta) -> "\e[35m"; +color_foreground(cyan) -> "\e[36m"; +color_foreground(white) -> "\e[37m". + +color_bold() -> "\e[1m". +reset_color() -> "\e[0m". From 8da0e014dcac75bff23ee827b139ea7fcaf08fd7 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 3 Jul 2015 12:10:48 +0200 Subject: [PATCH 09/89] Fix up colored log support * do not use pdict * do not enable color support by default. once we have the new getopt version, we can add a new type of command line flag for that. * fix Dialyzer warnings * use atom instead of boolean * use better name for internal function * do not try (and fail) to access rebar's app env in retest test --- ebin/rebar.app | 2 +- inttest/logging/logging_rt.erl | 26 ++++++----------- src/rebar_log.erl | 52 +++++++++++++++++----------------- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/ebin/rebar.app b/ebin/rebar.app index 985dbb96..710f8b26 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -69,7 +69,7 @@ {log_level, warn}, %% Log colored - {log_colored, true}, + {log_colored, uncolored}, %% any_dir processing modules {any_dir_modules, [ diff --git a/inttest/logging/logging_rt.erl b/inttest/logging/logging_rt.erl index 11cb0cdb..2709a84e 100644 --- a/inttest/logging/logging_rt.erl +++ b/inttest/logging/logging_rt.erl @@ -38,25 +38,9 @@ files() -> run(_Dir) -> SharedExpected = "==> logging_rt \\(compile\\)", + {ERROR, WARN, INFO, DEBUG} = log_labels(), %% provoke ERROR due to an invalid app file retest:log(info, "Check 'compile' failure output~n"), - {ERROR, WARN, INFO, DEBUG} = - case application:get_env(rebar, log_colored) of - {ok, true} -> - { - "\\e\\[1m\\e\\[31mERROR: \\e\\[0m", - "\\e\\[33mWARN: \\e\\[0m", - "\\e\\[32mINFO: \\e\\[0m", - "\\e\\[34mDEBUG: \\e\\[0m" - }; - _ -> - { - "ERROR: ", - "WARN: ", - "INFO: ", - "DEBUG: " - } - end, ok = check_output("./rebar compile -q", should_fail, [SharedExpected, ERROR], [WARN, INFO, DEBUG]), @@ -76,6 +60,14 @@ run(_Dir) -> [ERROR, INFO]), ok. +log_labels() -> + { + "(\\e\\[1m\\e\\[31mERROR: \\e\\[0m|ERROR: )", + "(\\e\\[33mWARN: \\e\\[0m|WARN: )", + "(\\e\\[32mINFO: \\e\\[0m|INFO: )", + "(\\e\\[34mDEBUG: \\e\\[0m|DEBUG: )" + }. + check_output(Cmd, FailureMode, Expected, Unexpected) -> case {retest:sh(Cmd), FailureMode} of {{error, _}=Error, should_succeed} -> diff --git a/src/rebar_log.erl b/src/rebar_log.erl index 91fb6596..abc2ac5a 100644 --- a/src/rebar_log.erl +++ b/src/rebar_log.erl @@ -43,6 +43,9 @@ %% Public API %% =================================================================== +%% TODO: Once we have the new getopt version, use a flag that +%% takes an optional arg but has a default value and fetch the +%% setting via rebar_config, after it has been set in rebar:main. init(Config) -> Verbosity = rebar_config:get_global(Config, verbose, default_level()), case valid_level(Verbosity) of @@ -50,27 +53,17 @@ init(Config) -> ?WARN_LEVEL -> set_level(warn); ?INFO_LEVEL -> set_level(info); ?DEBUG_LEVEL -> set_level(debug) - end, - LogColored = rebar_config:get_global(Config, log_colored, true), - set_log_colored(LogColored). - + end. set_level(Level) -> - erlang:put(rebar_log_level, Level). - -set_log_colored(true) -> - erlang:put(rebar_log_colored, true), - ok; -set_log_colored(_LogColored) -> - erlang:put(rebar_log_colored, false), - ok. + ok = application:set_env(rebar, log_level, Level). log(Level, Str, Args) -> log(standard_io, Level, Str, Args). log(Device, Level, Str, Args) -> - LogLevel = erlang:get(rebar_log_level), - LogColored = erlang:get(rebar_log_colored), + {ok, LogLevel} = application:get_env(rebar, log_level), + {ok, LogColored} = application:get_env(rebar, log_colored), case should_log(LogLevel, Level) of true -> io:format(Device, log_prefix(Level, LogColored) ++ Str, Args); @@ -101,33 +94,40 @@ should_log(error, error) -> true; should_log(error, _) -> false; should_log(_, _) -> false. -log_prefix(Level, _Colored = false) -> +log_prefix(Level, uncolored) -> log_prefix(Level); -log_prefix(Level, _Colored = true) -> - color_from_level(Level) ++ log_prefix(Level) ++ reset_color(). +log_prefix(Level, colored) -> + color_for_level(Level) ++ log_prefix(Level) ++ reset_color(). log_prefix(debug) -> "DEBUG: "; log_prefix(info) -> "INFO: "; log_prefix(warn) -> "WARN: "; log_prefix(error) -> "ERROR: ". -color_from_level(debug) -> +color_for_level(debug) -> color_foreground(blue); -color_from_level(info) -> +color_for_level(info) -> color_foreground(green); -color_from_level(warn) -> +color_for_level(warn) -> color_foreground(yellow); -color_from_level(error) -> +color_for_level(error) -> color_bold() ++ color_foreground(red). -color_foreground(black) -> "\e[30m"; +%% -type color() :: 'black' | 'red' | 'green' | 'yellow' +%% | 'blue' | 'magenta' | 'cyan' | 'white'. +%% -spec color_foreground(color()) -> string(). +%% +%% To silence Dialyzer, disable following colors, because they're +%% unused as of right now: black, magenta, cyan, white +%% +%% color_foreground(black) -> "\e[30m"; +%% color_foreground(magenta) -> "\e[35m"; +%% color_foreground(cyan) -> "\e[36m"; +%% color_foreground(white) -> "\e[37m"; color_foreground(red) -> "\e[31m"; color_foreground(green) -> "\e[32m"; color_foreground(yellow) -> "\e[33m"; -color_foreground(blue) -> "\e[34m"; -color_foreground(magenta) -> "\e[35m"; -color_foreground(cyan) -> "\e[36m"; -color_foreground(white) -> "\e[37m". +color_foreground(blue) -> "\e[34m". color_bold() -> "\e[1m". reset_color() -> "\e[0m". From 6ca71569c8f726ce24ba2396794aaf065384770f Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Tue, 14 Jul 2015 21:32:49 +0200 Subject: [PATCH 10/89] completion: add libid= to bash and zsh scripts --- priv/shell-completion/bash/rebar | 3 ++- priv/shell-completion/zsh/_rebar | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/priv/shell-completion/bash/rebar b/priv/shell-completion/bash/rebar index c017d20f..5889e17c 100644 --- a/priv/shell-completion/bash/rebar +++ b/priv/shell-completion/bash/rebar @@ -55,11 +55,12 @@ _rebar() jobs= \ suites= \ verbose=1 \ + libid= \ appid= \ + nodeid= \ overlay_vars= \ previous_release= \ profiler= \ - nodeid= \ root_dir= \ skip_deps=true \ skip_apps= \ diff --git a/priv/shell-completion/zsh/_rebar b/priv/shell-completion/zsh/_rebar index 2ba7cdc3..66a92640 100644 --- a/priv/shell-completion/zsh/_rebar +++ b/priv/shell-completion/zsh/_rebar @@ -63,11 +63,12 @@ _rebar () { 'jobs[Number of workers]::workers:(0 1 2 3 4 5 6 7 8 9)' \ 'suites[Common Test suites]::suite name:_path_files -W "(src test)" -g "*.erl(:r)"' \ 'verbose[Verbosity level]::verbosity level:(0 1 2 3)' \ + 'libid[Library id]:' \ 'appid[Application id]:' \ + 'nodeid[Node id]:' \ 'overlay_vars[Overlay variables file]:' \ 'previous_release[Previous release path]:' \ 'profiler[Select profiler]::flag:(fprof eflame)' \ - 'nodeid[Node id]:' \ 'root_dir[Reltool config root directory]::directory:_files -/' \ 'shell[Start a shell similar to erl -pa ebin -pa deps/*/ebin]' \ 'skip_deps[Skip deps]::flag:(true false)' \ From c9f7f274d678adc64bdf8214bd318c455d0fc400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Tue, 18 Aug 2015 23:46:01 +0100 Subject: [PATCH 11/89] Add regression test for covered common test Project has dependency that also contains a covered common test suite --- inttest/ct_cover/app.config | 2 ++ inttest/ct_cover/cover.spec | 3 ++ inttest/ct_cover/ct_cover_rt.erl | 30 +++++++++++++++++++ inttest/ct_cover/mock/dummy/app.config | 2 ++ inttest/ct_cover/mock/dummy/cover.spec | 3 ++ inttest/ct_cover/mock/dummy/ebin/dummy.app | 6 ++++ .../ct_cover/mock/dummy/itest/test_SUITE.erl | 17 +++++++++++ inttest/ct_cover/mock/dummy/rebar.config | 4 +++ inttest/ct_cover/rebar.config | 7 +++++ inttest/ct_cover/test_SUITE.erl | 17 +++++++++++ 10 files changed, 91 insertions(+) create mode 100644 inttest/ct_cover/app.config create mode 100644 inttest/ct_cover/cover.spec create mode 100644 inttest/ct_cover/ct_cover_rt.erl create mode 100644 inttest/ct_cover/mock/dummy/app.config create mode 100644 inttest/ct_cover/mock/dummy/cover.spec create mode 100644 inttest/ct_cover/mock/dummy/ebin/dummy.app create mode 100644 inttest/ct_cover/mock/dummy/itest/test_SUITE.erl create mode 100644 inttest/ct_cover/mock/dummy/rebar.config create mode 100644 inttest/ct_cover/rebar.config create mode 100644 inttest/ct_cover/test_SUITE.erl diff --git a/inttest/ct_cover/app.config b/inttest/ct_cover/app.config new file mode 100644 index 00000000..bb718b21 --- /dev/null +++ b/inttest/ct_cover/app.config @@ -0,0 +1,2 @@ +%% This file is an application config file, not a CT test config file +[{a1, [{foo, bar}]}]. diff --git a/inttest/ct_cover/cover.spec b/inttest/ct_cover/cover.spec new file mode 100644 index 00000000..353f3220 --- /dev/null +++ b/inttest/ct_cover/cover.spec @@ -0,0 +1,3 @@ +%% Cover analysis level. +%% Level = details | overview +{level, details}. diff --git a/inttest/ct_cover/ct_cover_rt.erl b/inttest/ct_cover/ct_cover_rt.erl new file mode 100644 index 00000000..c7f8776f --- /dev/null +++ b/inttest/ct_cover/ct_cover_rt.erl @@ -0,0 +1,30 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(ct_cover_rt). + +-compile(export_all). + +files() -> + [{create, "ebin/a1.app", app(a1)}, + {copy, "../../rebar", "rebar"}, + {copy, "rebar.config", "rebar.config"}, + {copy, "app.config", "app.config"}, + {copy, "cover.spec", "cover.spec"}, + {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}, + {copy, "mock", "deps"}]. + +run(_Dir) -> + {ok, _} = retest:sh("./rebar compile ct"), + ok. + +%% +%% Generate the contents of a simple .app file +%% +app(Name) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/inttest/ct_cover/mock/dummy/app.config b/inttest/ct_cover/mock/dummy/app.config new file mode 100644 index 00000000..c471954a --- /dev/null +++ b/inttest/ct_cover/mock/dummy/app.config @@ -0,0 +1,2 @@ +%% This file is an application config file, not a CT test config file +[{dummy, [{foo, bar}]}]. diff --git a/inttest/ct_cover/mock/dummy/cover.spec b/inttest/ct_cover/mock/dummy/cover.spec new file mode 100644 index 00000000..353f3220 --- /dev/null +++ b/inttest/ct_cover/mock/dummy/cover.spec @@ -0,0 +1,3 @@ +%% Cover analysis level. +%% Level = details | overview +{level, details}. diff --git a/inttest/ct_cover/mock/dummy/ebin/dummy.app b/inttest/ct_cover/mock/dummy/ebin/dummy.app new file mode 100644 index 00000000..3c89b1b3 --- /dev/null +++ b/inttest/ct_cover/mock/dummy/ebin/dummy.app @@ -0,0 +1,6 @@ +{application,dummy, + [{description,"dummy"}, + {vsn,"1"}, + {modules,[]}, + {registered,[]}, + {applications,[kernel,stdlib]}]}. diff --git a/inttest/ct_cover/mock/dummy/itest/test_SUITE.erl b/inttest/ct_cover/mock/dummy/itest/test_SUITE.erl new file mode 100644 index 00000000..0d9a065d --- /dev/null +++ b/inttest/ct_cover/mock/dummy/itest/test_SUITE.erl @@ -0,0 +1,17 @@ +-module(test_SUITE). + +-compile(export_all). + +-include_lib("ct.hrl"). + +all() -> + [simple_test, + app_config_file_test]. + +simple_test(Config) -> + io:format("Test: ~p\n", [Config]). + +app_config_file_test(_Config) -> + application:start(dummy), + {ok, bar} = application:get_env(dummy, foo), + application:stop(dummy). diff --git a/inttest/ct_cover/mock/dummy/rebar.config b/inttest/ct_cover/mock/dummy/rebar.config new file mode 100644 index 00000000..34b1c470 --- /dev/null +++ b/inttest/ct_cover/mock/dummy/rebar.config @@ -0,0 +1,4 @@ +{cover_enabled, true}. + +{ct_dir, "itest"}. +{ct_extra_params, "-repeat 2 -erl_args -config app -cover cover.spec"}. diff --git a/inttest/ct_cover/rebar.config b/inttest/ct_cover/rebar.config new file mode 100644 index 00000000..2b14ab8e --- /dev/null +++ b/inttest/ct_cover/rebar.config @@ -0,0 +1,7 @@ +{cover_enabled, true}. + +{deps_dir, "deps"}. +{deps,[{dummy, ".*"}]}. + +{ct_dir, "itest"}. +{ct_extra_params, "-repeat 2 -erl_args -config app"}. diff --git a/inttest/ct_cover/test_SUITE.erl b/inttest/ct_cover/test_SUITE.erl new file mode 100644 index 00000000..e8a2bb80 --- /dev/null +++ b/inttest/ct_cover/test_SUITE.erl @@ -0,0 +1,17 @@ +-module(test_SUITE). + +-compile(export_all). + +-include_lib("ct.hrl"). + +all() -> + [simple_test, + app_config_file_test]. + +simple_test(Config) -> + io:format("Test: ~p\n", [Config]). + +app_config_file_test(_Config) -> + application:start(a1), + {ok, bar} = application:get_env(a1, foo), + application:stop(a1). From 69802f63120c7bc87ad94eddeb43e1285ca770f9 Mon Sep 17 00:00:00 2001 From: Paulo Oliveira Date: Fri, 14 Aug 2015 01:01:41 +0100 Subject: [PATCH 12/89] Allow behaviors defined inside xref_extra_path Use case: xref'ing a module that defined behavior ranch_protocol was exiting with {'EXIT',{undef,[{ranch_protocol,behaviour_info, [callbacks],[]} --- inttest/xref_behavior/gen_xref_behavior.erl | 9 ++++++ inttest/xref_behavior/rebar.config | 6 ++++ inttest/xref_behavior/xref_behavior.erl | 10 +++++++ inttest/xref_behavior/xref_behavior_rt.erl | 32 +++++++++++++++++++++ src/rebar_xref.erl | 5 ++++ 5 files changed, 62 insertions(+) create mode 100644 inttest/xref_behavior/gen_xref_behavior.erl create mode 100644 inttest/xref_behavior/rebar.config create mode 100644 inttest/xref_behavior/xref_behavior.erl create mode 100644 inttest/xref_behavior/xref_behavior_rt.erl diff --git a/inttest/xref_behavior/gen_xref_behavior.erl b/inttest/xref_behavior/gen_xref_behavior.erl new file mode 100644 index 00000000..9ac66d29 --- /dev/null +++ b/inttest/xref_behavior/gen_xref_behavior.erl @@ -0,0 +1,9 @@ +-module(gen_xref_behavior). + +-export([behaviour_info/1]). -ignore_xref([{behaviour_info, 1}]). + +behaviour_info(callbacks) -> + [{init,1}, {handle, 1}]; +behaviour_info(_Other) -> + undefined. + diff --git a/inttest/xref_behavior/rebar.config b/inttest/xref_behavior/rebar.config new file mode 100644 index 00000000..c2e5af66 --- /dev/null +++ b/inttest/xref_behavior/rebar.config @@ -0,0 +1,6 @@ +{xref_warnings, true}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, exports_not_used, + deprecated_function_calls, deprecated_functions]}. + diff --git a/inttest/xref_behavior/xref_behavior.erl b/inttest/xref_behavior/xref_behavior.erl new file mode 100644 index 00000000..30b69592 --- /dev/null +++ b/inttest/xref_behavior/xref_behavior.erl @@ -0,0 +1,10 @@ +-module(xref_behavior). +-behavior(gen_xref_behavior). + +% behavior-defined callbacks don't require xref_ignore +-export([init/1, handle/1]). + +init(_Args) -> ok. + +handle(_Atom) -> next_event. + diff --git a/inttest/xref_behavior/xref_behavior_rt.erl b/inttest/xref_behavior/xref_behavior_rt.erl new file mode 100644 index 00000000..8536710b --- /dev/null +++ b/inttest/xref_behavior/xref_behavior_rt.erl @@ -0,0 +1,32 @@ +-module(xref_behavior_rt). + +-export([files/0, run/1]). + +files() -> + [ + {copy, "../../rebar", "rebar"}, + {copy, "rebar.config", "rebar.config"}, + {copy, "xref_behavior.erl", "src/xref_behavior.erl"}, + {copy, "gen_xref_behavior.erl", "src/gen_xref_behavior.erl"}, + {create, "ebin/xref_behavior.app", app(xref_behavior, + [xref_behavior, + gen_xref_behavior])} + ]. + +run(_Dir) -> + {ok, _} = retest_sh:run("./rebar compile", []), + {ok, _} = retest_sh:run("./rebar xref", []), + ok. + +%% +%% Generate the contents of a simple .app file +%% +app(Name, Modules) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, Modules}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). + diff --git a/src/rebar_xref.erl b/src/rebar_xref.erl index 5eb4499c..fdce75e8 100644 --- a/src/rebar_xref.erl +++ b/src/rebar_xref.erl @@ -59,6 +59,11 @@ xref(Config, _) -> OrigPath = code:get_path(), true = code:add_path(rebar_utils:ebin_dir()), + %% Add extra paths to code path to, for example, be used + %% when behaviour modules are defined + [code:add_path(Path) + || Path <- rebar_config:get(Config, xref_extra_paths, [])], + %% Get list of xref checks we want to run ConfXrefChecks = rebar_config:get(Config, xref_checks, [exports_not_used, From 41650ae66b4305b2f1638556e7126b536b86cb08 Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Mon, 24 Aug 2015 10:36:10 +0100 Subject: [PATCH 13/89] Fix eunit test on OTP18 OTP18 changed the output failed assertion string from assertEqual_failed to assertEqual --- test/rebar_eunit_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/rebar_eunit_tests.erl b/test/rebar_eunit_tests.erl index 33f168c5..a7fc51eb 100644 --- a/test/rebar_eunit_tests.erl +++ b/test/rebar_eunit_tests.erl @@ -131,7 +131,7 @@ eunit_with_suites_and_tests_test_() -> {"Selected suite's generator test raises an error", ?_assert(string:str(RebarOut, - "assertEqual_failed") =/= 0)}, + "assertEqual") =/= 0)}, {"Selected suite tests is run once", ?_assert(string:str(RebarOut, "Failed: 1.") =/= 0)}] From f9b850c428dae6cf6673ff076d863d5b6c343569 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Mon, 24 Aug 2015 19:39:15 +0200 Subject: [PATCH 14/89] Fix #536 --- src/rebar_port_compiler.erl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index dcd8a20a..10a4065c 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -628,11 +628,6 @@ default_env() -> {"solaris.*-64$", "CXXFLAGS", "-D_REENTRANT -m64 $CXXFLAGS"}, {"solaris.*-64$", "LDFLAGS", "-m64 $LDFLAGS"}, - %% Linux specific flags for multiarch - {"linux.*-64$", "CFLAGS", "-m64 $CFLAGS"}, - {"linux.*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, - {"linux.*-64$", "LDFLAGS", "$LDFLAGS"}, - %% OS X Leopard flags for 64-bit {"darwin9.*-64$", "CFLAGS", "-m64 $CFLAGS"}, {"darwin9.*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, From f7ecafbb0fb6989018431a6b1330de08aeeafd36 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 4 Sep 2015 16:56:10 +0200 Subject: [PATCH 15/89] Complete port compiler help string Move existing documentation from code comment into help string, so that 'rebar help compile' prints complete documentation for rebar_port_compiler. --- src/rebar_port_compiler.erl | 119 +++++++++++++++++------------------- 1 file changed, 56 insertions(+), 63 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index dcd8a20a..09329d5b 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -35,66 +35,16 @@ -include("rebar.hrl"). -%% =================================================================== -%% Public API -%% =================================================================== - -%% Supported configuration variables: -%% -%% * port_specs - Erlang list of tuples of the forms -%% {ArchRegex, TargetFile, Sources, Options} -%% {ArchRegex, TargetFile, Sources} -%% {TargetFile, Sources} -%% -%% * port_env - Erlang list of key/value pairs which will control -%% the environment when running the compiler and linker. -%% -%% By default, the following variables are defined: -%% CC - C compiler -%% CXX - C++ compiler -%% CFLAGS - C compiler -%% CXXFLAGS - C++ compiler -%% LDFLAGS - Link flags -%% ERL_CFLAGS - default -I paths for erts and ei -%% ERL_LDFLAGS - default -L and -lerl_interface -lei -%% DRV_CFLAGS - flags that will be used for compiling -%% DRV_LDFLAGS - flags that will be used for linking -%% EXE_CFLAGS - flags that will be used for compiling -%% EXE_LDFLAGS - flags that will be used for linking -%% ERL_EI_LIBDIR - ei library directory -%% DRV_CXX_TEMPLATE - C++ command template -%% DRV_CC_TEMPLATE - C command template -%% DRV_LINK_TEMPLATE - Linker command template -%% EXE_CXX_TEMPLATE - C++ command template -%% EXE_CC_TEMPLATE - C command template -%% EXE_LINK_TEMPLATE - Linker command template -%% PORT_IN_FILES - contains a space separated list of input -%% file(s), (used in command template) -%% PORT_OUT_FILE - contains the output filename (used in -%% command template) -%% -%% Note that if you wish to extend (vs. replace) these variables, -%% you MUST include a shell-style reference in your definition. -%% e.g. to extend CFLAGS, do something like: -%% -%% {port_env, [{"CFLAGS", "$CFLAGS -MyOtherOptions"}]} -%% -%% It is also possible to specify platform specific options -%% by specifying a triplet where the first string is a regex -%% that is checked against Erlang's system architecture string. -%% e.g. to specify a CFLAG that only applies to x86_64 on linux -%% do: -%% -%% {port_env, [{"x86_64.*-linux", "CFLAGS", -%% "$CFLAGS -X86Options"}]} -%% - -record(spec, {type::'drv' | 'exe', target::file:filename(), sources = [] :: [file:filename(), ...], objects = [] :: [file:filename(), ...], opts = [] ::list() | []}). +%% =================================================================== +%% Public API +%% =================================================================== + compile(Config, AppFile) -> case get_specs(Config, AppFile) of [] -> @@ -164,19 +114,62 @@ info_help(Description) -> "~s.~n" "~n" "Valid rebar.config options:~n" - " ~p~n" - " ~p~n" - "Cross-arch environment variables:~n" + "port_specs - Erlang list of tuples of the forms~n" + " {ArchRegex, TargetFile, Sources, Options}~n" + " {ArchRegex, TargetFile, Sources}~n" + " {TargetFile, Sources}~n" + "~n" + " Examples:~n" + " ~p~n" + "~n" + "port_env - Erlang list of key/value pairs which will control~n" + " the environment when running the compiler and linker.~n" + " Variables set in the surrounding system shell are taken~n" + " into consideration when expanding port_env.~n" + "~n" + " By default, the following variables are defined:~n" + " CC - C compiler~n" + " CXX - C++ compiler~n" + " CFLAGS - C compiler~n" + " CXXFLAGS - C++ compiler~n" + " LDFLAGS - Link flags~n" + " ERL_CFLAGS - default -I paths for erts and ei~n" + " ERL_LDFLAGS - default -L and -lerl_interface -lei~n" + " DRV_CFLAGS - flags that will be used for compiling~n" + " DRV_LDFLAGS - flags that will be used for linking~n" + " EXE_CFLAGS - flags that will be used for compiling~n" + " EXE_LDFLAGS - flags that will be used for linking~n" + " ERL_EI_LIBDIR - ei library directory~n" + " DRV_CXX_TEMPLATE - C++ command template~n" + " DRV_CC_TEMPLATE - C command template~n" + " DRV_LINK_TEMPLATE - Linker command template~n" + " EXE_CXX_TEMPLATE - C++ command template~n" + " EXE_CC_TEMPLATE - C command template~n" + " EXE_LINK_TEMPLATE - Linker command template~n" + "~n" + " Note that if you wish to extend (vs. replace) these variables,~n" + " you MUST include a shell-style reference in your definition.~n" + " e.g. to extend CFLAGS, do something like:~n" + "~n" + " {port_env, [{\"CFLAGS\", \"$CFLAGS -MyOtherOptions\"}]}~n" + "~n" + " It is also possible to specify platform specific options~n" + " by specifying a triplet where the first string is a regex~n" + " that is checked against Erlang's system architecture string.~n" + " e.g. to specify a CFLAG that only applies to x86_64 on linux~n" + " do:~n" + " {port_env, [{\"x86_64.*-linux\", \"CFLAGS\",~n" + " \"$CFLAGS -X86Options\"}]}~n" + "~n" + "Cross-arch environment variables to configure toolchain:~n" " REBAR_TARGET_ARCH to set the tool chain name to use~n" - " REBAR_TARGET_ARCH_WORDSIZE optional " - "(if CC fails to determine word size)~n" + " REBAR_TARGET_ARCH_WORDSIZE (optional - " + "if CC fails to determine word size)~n" " fallback word size is 32~n" - " REBAR_TARGET_ARCH_VSN optional " - "(if a special version of CC/CXX is requested)~n", + " REBAR_TARGET_ARCH_VSN (optional - " + "if a special version of CC/CXX is requested)~n", [ Description, - {port_env, [{"CFLAGS", "$CFLAGS -Ifoo"}, - {"freebsd", "LDFLAGS", "$LDFLAGS -lfoo"}]}, {port_specs, [{"priv/so_name.so", ["c_src/*.c"]}, {"linux", "priv/hello_linux", ["c_src/hello_linux.c"]}, {"linux", "priv/hello_linux", ["c_src/*.c"], [{env, []}]}]} From e025430da864374c16a9c59f0c33fd8438213294 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 11 Sep 2015 20:00:32 +0200 Subject: [PATCH 16/89] rmemo: use a better R13 check (Reported-by: Stavros Aronis) --- src/rmemo.erl | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/rmemo.erl b/src/rmemo.erl index dd3642f3..7de29fc8 100644 --- a/src/rmemo.erl +++ b/src/rmemo.erl @@ -184,18 +184,14 @@ init(_) -> -spec ets_tab() -> ets:tab(). ets_tab() -> - ErtsApp = filename:join(code:lib_dir(erts, ebin), "erts.app"), Concurrency = - %% If erts.app exists, we run on at least R14. That means we - %% can use ets read_concurrency. - %% TODO: Remove and revert to vanilla memo.erl from - %% https://github.com/tuncer/memo once we require at least - %% R14B and drop support for R13. - case filelib:is_regular(ErtsApp) of + %% read_concurrency was added in R14. If we're running R13, + %% do not try to use it. + case erlang:system_info(version) =< "5.8.3" of true -> - [{read_concurrency, true}]; + []; false -> - [] + [{read_concurrency, true}] end, ets:new( ?TABLE, From c376ef688e5944fc2133c442beafc4f8247d1ee4 Mon Sep 17 00:00:00 2001 From: Cesar Crusius Date: Thu, 17 Sep 2015 08:33:28 -0700 Subject: [PATCH 17/89] Fix #544 'whoami' is not POSIX-compliant, but 'id -un' is. This is true at least since the 1003.1-2001 standard, I don't have a copy of the previous 1003.2-1992 to verify it is true there too. This causes problems in recent operating systems where 'whoami' is not even shipped anymore. --- priv/templates/simplenode.runner | 5 +---- test/upgrade_project/rel/files/dummy | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/priv/templates/simplenode.runner b/priv/templates/simplenode.runner index a6055130..887867b1 100755 --- a/priv/templates/simplenode.runner +++ b/priv/templates/simplenode.runner @@ -6,9 +6,6 @@ if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then POSIX_SHELL="true" export POSIX_SHELL - # To support 'whoami' add /usr/ucb to path - PATH=/usr/ucb:$PATH - export PATH exec /usr/bin/ksh $0 "$@" fi @@ -25,7 +22,7 @@ RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc # Note the trailing slash on $PIPE_DIR/ PIPE_DIR=/tmp/$RUNNER_BASE_DIR/ RUNNER_USER= -WHOAMI=$(whoami) +WHOAMI=$(id -un) # Make sure this script is running as the appropriate user if ([ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]); then diff --git a/test/upgrade_project/rel/files/dummy b/test/upgrade_project/rel/files/dummy index 67fb2da7..7b2caa1f 100755 --- a/test/upgrade_project/rel/files/dummy +++ b/test/upgrade_project/rel/files/dummy @@ -6,9 +6,6 @@ if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then POSIX_SHELL="true" export POSIX_SHELL - # To support 'whoami' add /usr/ucb to path - PATH=/usr/ucb:$PATH - export PATH exec /usr/bin/ksh $0 "$@" fi @@ -24,7 +21,7 @@ RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc # Note the trailing slash on $PIPE_DIR/ PIPE_DIR=/tmp/$RUNNER_BASE_DIR/ RUNNER_USER= -WHOAMI=$(whoami) +WHOAMI=$(id -un) # Make sure this script is running as the appropriate user if ([ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]); then From 5119baf9cb8327f9f682f9446d923a1a3cf89a13 Mon Sep 17 00:00:00 2001 From: Cesar Crusius Date: Fri, 18 Sep 2015 08:43:52 -0700 Subject: [PATCH 18/89] Fix #544 even more. A bunch of fixes: * Only try to find out the user that is running the script if that's necessary. That allows us to error out if we can't find the user name. * Fallback to 'whoami' in the unlikely case that 'id -un' does not work. * Use 'su' if 'sudo' is not installed, and if the user is 'root'. (The 'sudo' binary is not installed by default in many OSs.) --- priv/templates/simplenode.runner | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/priv/templates/simplenode.runner b/priv/templates/simplenode.runner index 887867b1..ba438174 100755 --- a/priv/templates/simplenode.runner +++ b/priv/templates/simplenode.runner @@ -22,17 +22,31 @@ RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc # Note the trailing slash on $PIPE_DIR/ PIPE_DIR=/tmp/$RUNNER_BASE_DIR/ RUNNER_USER= -WHOAMI=$(id -un) # Make sure this script is running as the appropriate user -if ([ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]); then - type sudo > /dev/null 2>&1 +if [ "$RUNNER_USER" ]; then + WHOAMI=$(id -un 2>/dev/null || whoami 2>/dev/null) if [ $? -ne 0 ]; then - echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 + echo "Could not determine user name." exit 1 fi - echo "Attempting to restart script through sudo -H -u $RUNNER_USER" >&2 - exec sudo -H -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ + if [ "x$WHOAMI" != "x$RUNNER_USER" ]; then + # The 'su' command is more portable, but can't be configured as 'sudo' + # can to allow non-interactive calls from non-root users. + type sudo > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "Attempting to restart script through sudo -H -u $RUNNER_USER" >&2 + exec sudo -H -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ + else + if [ "x$WHOAMI" != "xroot" ]; then + echo "Only root can run $RUNNER_SCRIPT as $RUNNER_USER without requiring a password." + exit 1 + else + echo "Attempting to restart script through su $RUNNER_USER" >&2 + exec su $RUNNER_USER -c $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ + fi + fi + fi fi # Identify the script name From 65ee732d71308e3f79fbb828215f3510cfb66144 Mon Sep 17 00:00:00 2001 From: Cesar Crusius Date: Fri, 18 Sep 2015 12:01:51 -0700 Subject: [PATCH 19/89] Fix #544 Bring back Solaris' ucb PATH just in case. --- priv/templates/simplenode.runner | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/priv/templates/simplenode.runner b/priv/templates/simplenode.runner index ba438174..814b1128 100755 --- a/priv/templates/simplenode.runner +++ b/priv/templates/simplenode.runner @@ -6,6 +6,10 @@ if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then POSIX_SHELL="true" export POSIX_SHELL + # To support 'whoami' on old Solaris systems, add /usr/ucb to path + # (New Solaris systems do not have 'whoami') + PATH=/usr/ucb:$PATH + export PATH exec /usr/bin/ksh $0 "$@" fi From a7e83237a478523f3ffe5cad55089e603c2556a0 Mon Sep 17 00:00:00 2001 From: Andrey Teplyashin Date: Wed, 26 Aug 2015 13:07:24 +0600 Subject: [PATCH 20/89] Add erl_first_files to other first_files --- src/rebar_erlc_compiler.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 5c332398..412b29d0 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -236,8 +236,10 @@ test_compile_config_and_opts(Config, ErlOpts, Cmd) -> Opts = [O || O <- Opts0, O =/= no_debug_info], Config4 = rebar_config:set(Config3, erl_opts, Opts), + FirstErlFiles = rebar_config:get_list(Config4, erl_first_files, []), FirstFilesAtom = list_to_atom(Cmd ++ "_first_files"), - FirstErls = rebar_config:get_list(Config4, FirstFilesAtom, []), + FirstErlsOther = rebar_config:get_list(Config4, FirstFilesAtom, []), + FirstErls = FirstErlFiles ++ FirstErlsOther, Config5 = rebar_config:set(Config4, erl_first_files, FirstErls), {Config5, Opts}. From 06c4b5df68095b598b08600c17090b3e822492df Mon Sep 17 00:00:00 2001 From: Andrey Teplyashin Date: Tue, 15 Sep 2015 22:21:04 +0600 Subject: [PATCH 21/89] Add test for eunit_first_files --- test/rebar_eunit_tests.erl | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/test/rebar_eunit_tests.erl b/test/rebar_eunit_tests.erl index a7fc51eb..cb331a4e 100644 --- a/test/rebar_eunit_tests.erl +++ b/test/rebar_eunit_tests.erl @@ -231,7 +231,21 @@ eunit_with_suites_and_tests_test_() -> [?_assert(string:str(RebarOut, "Failed: 1. Skipped: 0. Passed: 1") =/= 0)]}] - end}]. + end}, + {"Ensure EUnit runs a test with eunit_first_files", + setup, + fun() -> + setup_eunit_first_files(), + rebar("eunit") + end, + fun teardown/1, + fun(RebarOut) -> + [ + {"Don't pass tests without erl_first_file", + [?_assert(string:str(RebarOut, + "Test passed.") =/= 0)]}] + end} + ]. cover_test_() -> {"Ensure Cover runs with tests in a test dir and no defined suite", @@ -420,6 +434,19 @@ code_path_test_() -> "myfunc3() -> ok.\n", "mygenerator_test_() -> [?_assertEqual(true, false)].\n"]). +-define(myapp_mymod4, + ["-module(myapp_mymod4).\n", + "-compile({parse_transform, myapp_mymod4_parse_transform}).\n", + "-include_lib(\"eunit/include/eunit.hrl\").\n", + "-export([ok/0]).\n", + "pt_test() -> ?assert(myapp_mymod4:ok()).\n"]). + +-define(myapp_mymod4_parse_transform, + ["-module(myapp_mymod4_parse_transform).\n", + "-export([parse_transform/2]).\n", + "parse_transform(Forms, _Options) ->\n", + "Forms ++ [{function,29,ok,0,[{clause,9,[],[],[{atom,9,true}]}]}].\n"]). + -define(mysuite, ["-module(mysuite).\n", "-export([all_test_/0]).\n", @@ -462,6 +489,15 @@ setup_project_with_multiple_modules() -> ok = file:write_file("src/myapp_mymod2.erl", ?myapp_mymod2), ok = file:write_file("src/myapp_mymod3.erl", ?myapp_mymod3). +setup_eunit_first_files() -> + setup_environment(), + rebar("create-app appid=myapp"), + ok = file:write_file("src/myapp_mymod4.erl", ?myapp_mymod4), + ok = file:write_file("src/myapp_mymod4_parse_transform.erl", + ?myapp_mymod4_parse_transform), + ok = file:write_file("rebar.config", + "{erl_first_files, [\"src/myapp_mymod4_parse_transform.erl\"]}.\n"). + setup_cover_project() -> setup_basic_project(), ok = file:write_file("rebar.config", "{cover_enabled, true}.\n"). From b32aaae5e77f457110732b3fea586d865c9371fc Mon Sep 17 00:00:00 2001 From: "Duncan M. McGreggor" Date: Sat, 26 Sep 2015 12:18:36 -0500 Subject: [PATCH 22/89] Add report and verbose options for LFE compile Recent builds of LFE have stopped reporting compile errors when compiled with rebar. Examination of this issue revealed that this was due to recent changes in default LFE compile options. The way that LFE handles compile options has slightly changed; ``verbose`` and ``report`` are now provided as defaults only if no other options are passed; if other options are passed, these do not get set. As a result, we have stopped seeing compile error info when building with rebar. This change brings back the previous level of reporting that LFE developers had when compiling their projects with rebar. --- THANKS | 1 + src/rebar_lfe_compiler.erl | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 7a5f5ecd..582a1530 100644 --- a/THANKS +++ b/THANKS @@ -144,3 +144,4 @@ Stavros Aronis James Fish Tony Rogvall Andrey Teplyashin +Duncan McGreggor diff --git a/src/rebar_lfe_compiler.erl b/src/rebar_lfe_compiler.erl index 8488b0ff..7d37914a 100644 --- a/src/rebar_lfe_compiler.erl +++ b/src/rebar_lfe_compiler.erl @@ -71,7 +71,9 @@ compile_lfe(Source, _Target, Config) -> ?FAIL; _ -> ErlOpts = rebar_utils:erl_opts(Config), - Opts = [{i, "include"}, {outdir, "ebin"}, return] ++ ErlOpts, + LfeOpts = [report, verbose, {i, "include"}, {outdir, "ebin"}, + return], + Opts = LfeOpts ++ ErlOpts, case lfe_comp:file(Source, Opts) of {ok, _Mod, Ws} -> rebar_base_compiler:ok_tuple(Config, Source, Ws); From 365ac649dc818619757f96a699ddb174f004cff9 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 28 Sep 2015 08:23:36 -0400 Subject: [PATCH 23/89] bump to 2.6.1 --- RELEASE-NOTES.md | 18 ++++++++++++++++++ ebin/rebar.app | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index aa695552..329575f4 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,21 @@ +# 2.6.1 + +* rebar/514: [Add license to relnotes tool per reviewer request](https://github.com/rebar/rebar/pull/514) +* rebar/518: [Generate reserved file for erlc regression test](https://github.com/rebar/rebar/pull/518) +* rebar/520: [Fixed version of #451](https://github.com/rebar/rebar/pull/520) +* rebar/521: [rebar_utils: fix comment](https://github.com/rebar/rebar/pull/521) +* rebar/522: [Fixed version of #395](https://github.com/rebar/rebar/pull/522) +* rebar/524: [completion: add libid= to bash and zsh scripts](https://github.com/rebar/rebar/pull/524) +* rebar/530: [Allow behaviors defined inside xref_extra_path](https://github.com/rebar/rebar/pull/530) +* rebar/535: [Add regression test for covered common test](https://github.com/rebar/rebar/pull/535) +* rebar/537: [Fix eunit test on OTP18](https://github.com/rebar/rebar/pull/537) +* rebar/538: [Fix #536](https://github.com/rebar/rebar/pull/538) +* rebar/539: [Add erl_first_files in eunit_first_files and qc_first_files](https://github.com/rebar/rebar/pull/539) +* rebar/540: [Complete port compiler help string](https://github.com/rebar/rebar/pull/540) +* rebar/543: [rmemo: use a better R13 check (Reported-by: Stavros Aronis)](https://github.com/rebar/rebar/pull/543) +* rebar/545: [Fix #544](https://github.com/rebar/rebar/pull/545) +* rebar/546: [Added report and verbose options for LFE compile.](https://github.com/rebar/rebar/pull/546) + # 2.6.0 * rebar/203: [Pluggable proto compilers gpb](https://github.com/rebar/rebar/pull/203) diff --git a/ebin/rebar.app b/ebin/rebar.app index 710f8b26..b4de6162 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "2.6.0"}, + {vsn, "2.6.1"}, {modules, [ rebar, rebar_abnfc_compiler, rebar_app_utils, From e9983fc422443f68e2fafc7cfef5619ed249bb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Mon, 12 Oct 2015 13:46:06 +0100 Subject: [PATCH 24/89] Add Windows continuous integration through AppVeyor --- README.md | 1 + appveyor.yml | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 appveyor.yml diff --git a/README.md b/README.md index ec656934..a7317baa 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ rebar is an Erlang build tool that makes it easy to compile and test Erlang applications, port drivers and releases. [![Build Status](https://secure.travis-ci.org/rebar/rebar.png?branch=master)](http://travis-ci.org/rebar/rebar) +[![Build status](https://ci.appveyor.com/api/projects/status/4c48dw0cbu265cvt?svg=true)](https://ci.appveyor.com/project/rebar-win-ci/rebar-t0g71) rebar is a self-contained Erlang script, so it's easy to distribute or even embed directly in a project. Where possible, rebar uses standard Erlang/OTP diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..a13f3d61 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,13 @@ +version: 2.6.1.{build} +branches: + only: + - master +build_script: +- cmd: bootstrap.bat +test_script: +- cmd: >- + rebar eunit + + deps/retest/retest -l debug inttest +artifacts: +- path: rebar From 7045f8efed071573992b43ef7df6aadff5678c70 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Wed, 14 Oct 2015 11:23:25 +0200 Subject: [PATCH 25/89] fprof: document Cachegrind support --- src/rebar.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rebar.erl b/src/rebar.erl index dcfb3531..3acd73ad 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -501,7 +501,9 @@ option_spec_list() -> {profile, $p, "profile", undefined, "Profile this run of rebar. Via profiler= you can optionally select " "either fprof (default) or eflame. The result can be found in " - "fprof.analysis or eflame.svg."}, + "fprof.analysis or eflame.svg. Additionally, in fprof mode, if " + "erlgrind can be found in $PATH, a Cachegrind file will be generated " + "as well."}, {keep_going, $k, "keep-going", undefined, "Keep running after a command fails"}, {recursive, $r, "recursive", boolean, From 52f4f6822214bdd9ed79dd32c24299040871b658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Mon, 19 Oct 2015 15:13:00 +0100 Subject: [PATCH 26/89] fprof: further document Cachegrind support --- src/rebar.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rebar.erl b/src/rebar.erl index 3acd73ad..2fceb195 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -502,8 +502,8 @@ option_spec_list() -> "Profile this run of rebar. Via profiler= you can optionally select " "either fprof (default) or eflame. The result can be found in " "fprof.analysis or eflame.svg. Additionally, in fprof mode, if " - "erlgrind can be found in $PATH, a Cachegrind file will be generated " - "as well."}, + "erlgrind can be found in $PATH, a Cachegrind file (fprof.cgrind) " + "will be generated as well."}, {keep_going, $k, "keep-going", undefined, "Keep running after a command fails"}, {recursive, $r, "recursive", boolean, From b514e253599a2970f6a8acde64677fe04a0d2434 Mon Sep 17 00:00:00 2001 From: Sergey Savenko Date: Sun, 25 Oct 2015 16:57:51 +0300 Subject: [PATCH 27/89] Add qualified name tests docs (see pr #119) --- src/rebar_eunit.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index 913f2f9c..f1e2ac38 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -128,6 +128,7 @@ info_help(Description) -> " name starts with bar and, if no such test exists,~n" " run the test whose name starts with bar in the~n" " suite's _tests module)~n" + " test[s]=\"foo:bar_test\" (Run bar_test located in module foo)~n" " random_suite_order=true (Run tests in random order)~n" " random_suite_order=Seed (Run tests in random order,~n" " with the PRNG seeded with Seed)~n" From ea84fdaf31fddab55ba3d9630710b21e5db209bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Mon, 12 Oct 2015 13:48:09 +0100 Subject: [PATCH 28/89] Fix windows eunit tests File tests: windows file operations should abort on error the same as unix operations invoked through sh. Also windows does not support the '?' character in filenames. Eunit tests: the 'All' prefix is missing on the 'x tests passed' message. Eunit only prints the 'All' prefix if there are more than 2 passed tests, dropping the prefix on the match works for all cases. --- src/rebar_file_utils.erl | 12 +++++++----- test/rebar_compiler_tests.erl | 6 +++--- test/rebar_eunit_tests.erl | 16 ++++++++++------ test/rebar_file_utils_tests.erl | 2 +- test/rebar_xref_eunit.erl | 4 ++-- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 0fc1403e..a4d3be75 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -88,7 +88,7 @@ mv(Source, Dest) -> ?FMT("move /y \"~s\" \"~s\" 1> nul", [filename:nativename(Source), filename:nativename(Dest)]), - [{use_stdout, false}, return_on_error]), + [{use_stdout, false}, abort_on_error]), case R of [] -> ok; @@ -131,14 +131,14 @@ delete_each_dir_win32([]) -> ok; delete_each_dir_win32([Dir | Rest]) -> {ok, []} = rebar_utils:sh(?FMT("rd /q /s \"~s\"", [filename:nativename(Dir)]), - [{use_stdout, false}, return_on_error]), + [{use_stdout, false}, abort_on_error]), delete_each_dir_win32(Rest). xcopy_win32(Source,Dest)-> {ok, R} = rebar_utils:sh( ?FMT("xcopy \"~s\" \"~s\" /q /y /e 2> nul", [filename:nativename(Source), filename:nativename(Dest)]), - [{use_stdout, false}, return_on_error]), + [{use_stdout, false}, abort_on_error]), case length(R) > 0 of %% when xcopy fails, stdout is empty and and error message is printed %% to stderr (which is redirected to nul) @@ -162,8 +162,10 @@ cp_r_win32({false, Source} = S,{true, DestDir}) -> cp_r_win32(S, {false, filename:join(DestDir, filename:basename(Source))}); cp_r_win32({false, Source},{false, Dest}) -> %% from file to file - {ok,_} = file:copy(Source, Dest), - ok; + case file:copy(Source, Dest) of + {ok,_} -> ok; + _ -> throw(rebar_abort) + end; cp_r_win32({true, SourceDir}, {false, DestDir}) -> case filelib:is_regular(DestDir) of true -> diff --git a/test/rebar_compiler_tests.erl b/test/rebar_compiler_tests.erl index 3ed600b6..104a7d88 100644 --- a/test/rebar_compiler_tests.erl +++ b/test/rebar_compiler_tests.erl @@ -84,11 +84,11 @@ not_keep_going_test_() -> setup, fun() -> setup_basic_project(), - setup_rebar_config(), - rebar("compile") + setup_rebar_config() end, fun teardown/1, - fun(RebarOut)-> + fun()-> + RebarOut = rebar("compile"), [ {"Exit after error", ?_assert(string:str(RebarOut, "ERROR: compile failed") =/= 0)} diff --git a/test/rebar_eunit_tests.erl b/test/rebar_eunit_tests.erl index cb331a4e..d481daee 100644 --- a/test/rebar_eunit_tests.erl +++ b/test/rebar_eunit_tests.erl @@ -56,7 +56,7 @@ eunit_test_() -> ?_assert(string:str(RebarOut, "myapp_mymod:") =/= 0)}, {"Tests are only run once", - ?_assert(string:str(RebarOut, "All 2 tests passed") =/= 0)}] + ?_assert(string:str(RebarOut, "2 tests passed") =/= 0)}] end}. eunit_with_suites_and_tests_test_() -> @@ -80,7 +80,7 @@ eunit_with_suites_and_tests_test_() -> ?_assert(string:str(RebarOut, "myapp_mymod:") =:= 0)}, {"Selected suite tests are only run once", - ?_assert(string:str(RebarOut, "All 4 tests passed") =/= 0)}] + ?_assert(string:str(RebarOut, "4 tests passed") =/= 0)}] end}, {"Ensure EUnit runs selected _tests suites", setup, fun() -> @@ -102,7 +102,7 @@ eunit_with_suites_and_tests_test_() -> ?_assert(string:str(RebarOut, "myapp_mymod:") =:= 0)}, {"Selected suite tests are only run once", - ?_assert(string:str(RebarOut, "All 2 tests passed") =/= 0)}] + ?_assert(string:str(RebarOut, "2 tests passed") =/= 0)}] end}, {"Ensure EUnit runs a specific test defined in a selected suite", setup, fun() -> @@ -154,7 +154,7 @@ eunit_with_suites_and_tests_test_() -> "myapp_mymod2_tests:myfunc2_test/0") =/= 0)]}, {"Selected suite tests are run once", - ?_assert(string:str(RebarOut, "All 3 tests passed") =/= 0)}] + ?_assert(string:str(RebarOut, "3 tests passed") =/= 0)}] end}, {"Ensure EUnit runs specific test in a _tests suite", setup, @@ -190,7 +190,7 @@ eunit_with_suites_and_tests_test_() -> =/= 0)]}, {"Selected suite tests is run once", - ?_assert(string:str(RebarOut, "All 2 tests passed") =/= 0)}] + ?_assert(string:str(RebarOut, "2 tests passed") =/= 0)}] end}, {"Ensure EUnit runs a specific test by qualified function name", setup, @@ -325,7 +325,11 @@ environment_test_() -> assert_rebar_runs() -> prepare_rebar_script(), - ?assert(string:str(os:cmd(filename:nativename("./" ++ ?TMP_DIR ++ "rebar")), + {ok, Cwd} = file:get_cwd(), + ok = file:set_cwd(?TMP_DIR), + RebarOut = os:cmd(filename:nativename("./rebar")), + ok = file:set_cwd(Cwd), + ?assert(string:str(RebarOut, "No command to run specified!") =/= 0). basic_setup_test_() -> diff --git a/test/rebar_file_utils_tests.erl b/test/rebar_file_utils_tests.erl index fc76d589..c9b4192c 100644 --- a/test/rebar_file_utils_tests.erl +++ b/test/rebar_file_utils_tests.erl @@ -36,7 +36,7 @@ -define(TMP_DIR, "tmp_file_utils"). --define(SRC, "source dir?"). +-define(SRC, "source dir"). -define(DST, "dest (dir)"). -define(FILE1, "file 1"). -define(FILE2, "file(2)"). diff --git a/test/rebar_xref_eunit.erl b/test/rebar_xref_eunit.erl index 341fe2e1..f32ea46b 100644 --- a/test/rebar_xref_eunit.erl +++ b/test/rebar_xref_eunit.erl @@ -192,8 +192,8 @@ prepare_rebar_script() -> {unix, _} -> [] = os:cmd("chmod u+x " ++ Rebar); {win32, _} -> - {ok, _} = file:copy(?REBAR_SCRIPT ++ ".bat", - ?TMP_DIR ++ "rebar.bat") + {ok, _} = file:copy(?REBAR_SCRIPT ++ ".cmd", + ?TMP_DIR ++ "rebar.cmd") end. rebar() -> From 13b8d067315d768775360792810f4fd530e6c718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Mon, 2 Nov 2015 09:40:52 +0000 Subject: [PATCH 29/89] Lock retest dependency Master right now is broken, prevents rebar integration tests from succeeding. --- rebar.config.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.script b/rebar.config.script index 67356453..25e3f807 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,7 +1,7 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 ft=erlang et -ExtraDeps = [{retest, ".*", {git, "git://github.com/dizzyd/retest.git"}}], +ExtraDeps = [{retest, ".*", {git, "git://github.com/dizzyd/retest.git", {tag, "4590941a"}}}], case os:getenv("REBAR_EXTRA_DEPS") of false -> From c615a8d08198d48c814dc45c0ce7a4d3a64f0660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Serre?= Date: Mon, 16 Nov 2015 19:51:05 +0100 Subject: [PATCH 30/89] Allow windows release to use etc conf dir --- THANKS | 1 + priv/templates/simplenode.windows.runner.cmd | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/THANKS b/THANKS index 582a1530..0d4b5ac1 100644 --- a/THANKS +++ b/THANKS @@ -145,3 +145,4 @@ James Fish Tony Rogvall Andrey Teplyashin Duncan McGreggor +Sebastien Serre diff --git a/priv/templates/simplenode.windows.runner.cmd b/priv/templates/simplenode.windows.runner.cmd index d71a8c7d..1e3a3f3e 100644 --- a/priv/templates/simplenode.windows.runner.cmd +++ b/priv/templates/simplenode.windows.runner.cmd @@ -17,8 +17,18 @@ @call :set_trim release_version %%J ) -@set vm_args=%releases_dir%\%release_version%\vm.args -@set sys_config=%releases_dir%\%release_version%\sys.config +@if exist "%releases_dir%\%release_version%\vm.args" ( + @set vm_args="%releases_dir%\%release_version%\vm.args" +) else ( + @set vm_args="%node_root%\etc\vm.args" +) + +@if exist "%releases_dir%\%release_version%\sys.config" ( + @set sys_config="%releases_dir%\%release_version%\sys.config" +) else ( + @set sys_config="%node_root%\etc\app.config" +) + @set node_boot_script=%releases_dir%\%release_version%\%node_name% @set clean_boot_script=%releases_dir%\%release_version%\start_clean From 0661066cb04b80707c10ca9fd82690f8e581629c Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sat, 28 Nov 2015 00:37:48 +0100 Subject: [PATCH 31/89] xref: fix dialyzer warning introduced in 69802f63120 --- src/rebar_xref.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rebar_xref.erl b/src/rebar_xref.erl index fdce75e8..ee3414dc 100644 --- a/src/rebar_xref.erl +++ b/src/rebar_xref.erl @@ -60,9 +60,9 @@ xref(Config, _) -> true = code:add_path(rebar_utils:ebin_dir()), %% Add extra paths to code path to, for example, be used - %% when behaviour modules are defined - [code:add_path(Path) - || Path <- rebar_config:get(Config, xref_extra_paths, [])], + %% when behaviour modules are defined. + lists:foreach(fun(P) -> true = code:add_path(P) end, + rebar_config:get(Config, xref_extra_paths, [])), %% Get list of xref checks we want to run ConfXrefChecks = rebar_config:get(Config, xref_checks, From ac4451ff235ded6c4358f00fe9a57b219cf3b418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Sun, 13 Dec 2015 17:11:59 +0000 Subject: [PATCH 32/89] Provide additional debug logging on relup generation --- src/rebar_upgrade.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rebar_upgrade.erl b/src/rebar_upgrade.erl index 23da5a31..01edbd29 100644 --- a/src/rebar_upgrade.erl +++ b/src/rebar_upgrade.erl @@ -44,18 +44,24 @@ 'generate-upgrade'(Config0, ReltoolFile) -> %% Get the old release path {Config, ReltoolConfig} = rebar_rel_utils:load_config(Config0, ReltoolFile), + ?DEBUG("reltool.config: ~p~n", [ReltoolConfig]), TargetParentDir = rebar_rel_utils:get_target_parent_dir(Config, ReltoolConfig), TargetDir = rebar_rel_utils:get_target_dir(Config, ReltoolConfig), + ?DEBUG("target dir: ~p~n", [TargetDir]), PrevRelPath = rebar_rel_utils:get_previous_release_path(Config), OldVerPath = filename:join([TargetParentDir, PrevRelPath]), + ?DEBUG("old version path: ~p~n", [OldVerPath]), %% Run checks to make sure that building a package is possible {NewVerPath, NewName, NewVer, OldVer} = run_checks(Config, OldVerPath, ReltoolConfig), + ?DEBUG("old version: ~p~n", [OldVer]), NameVer = NewName ++ "_" ++ NewVer, OldRelName = get_old_rel_name(OldVerPath, OldVer, NewName), + ?DEBUG("new version path: ~p~n", [NewVerPath]), + ?DEBUG("old version: ~p~n", [NewVer]), %% Save the code path prior to doing anything OrigPath = code:get_path(), From 0bf08e4e34469144ae6f3297aae4a38b3bb2bd3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Sun, 25 Oct 2015 16:14:28 +0000 Subject: [PATCH 33/89] Add support for Windows integration testing Use retest feature/rebar-windows-ci branch that adds Windows tests support, test setup callback and additional touch command. For all tests copy rebar and rebar.cmd using retest setup callback. Port OS specific commands used in tests to Erlang (eg. touch, rm, cp, stat..). rebar_ct: do away with grep command line invocation (which doesn't exist in Windows) and use instead plain Erlang parsing. Increase timeout for rgen1 test to 4 minutes, Windows Appveyor can take longer than the previous 2 minutes. --- appveyor.yml | 21 +++-- inttest/app_src/app_src_rt.erl | 9 +- inttest/app_src_script/app_src_script_rt.erl | 9 +- .../app_src_script_2/app_src_script_2_rt.erl | 9 +- inttest/appup_src/appup_src_rt.erl | 9 +- inttest/appup_src_2/appup_src_2_rt.erl | 9 +- inttest/appup_src_2/appup_src_rt_2.erl | 8 +- .../appup_src_script/appup_src_script_rt.erl | 9 +- inttest/bug_5_rt.erl | 9 +- .../code_path_no_recurse_rt.erl | 7 +- inttest/cover/cover_rt.erl | 11 ++- inttest/ct1/ct1_rt.erl | 12 +-- inttest/ct2/ct2_rt.erl | 10 ++- inttest/ct3/ct3_rt.erl | 12 ++- inttest/ct_cover/ct_cover_rt.erl | 11 ++- inttest/depplugins/depplugins_rt.erl | 7 +- inttest/erlc/erlc_rt.erl | 7 +- inttest/erlc_dep_graph/erlc_dep_graph_rt.erl | 30 +++++-- inttest/eunit/eunit_rt.erl | 11 ++- inttest/inttest_utils.erl | 16 ++++ inttest/logging/logging_rt.erl | 7 +- inttest/port/port_rt.erl | 87 +++++++++++++------ inttest/port/rebar.config | 3 +- inttest/profile/profile_rt.erl | 8 +- inttest/proto_gpb/proto_gpb_rt.erl | 14 +-- .../proto_protobuffs/proto_protobuffs_rt.erl | 7 +- inttest/require_vsn/require_vsn_rt.erl | 7 +- inttest/rgen1/retest.config | 2 +- inttest/rgen1/rgen1_rt.erl | 9 +- .../t_custom_config/t_custom_config_rt.erl | 9 +- inttest/tdeps1/tdeps1_rt.erl | 10 ++- inttest/tdeps2/tdeps2_rt.erl | 10 ++- inttest/tdeps3/tdeps3_rt.erl | 10 ++- inttest/tdeps_update/tdeps_update_rt.erl | 75 +++++++++------- inttest/thooks/thooks_rt.erl | 15 +++- inttest/tplugins/tplugins_rt.erl | 7 +- inttest/xref_behavior/xref_behavior_rt.erl | 7 +- rebar.config.script | 3 +- src/rebar_ct.erl | 35 ++++++-- 39 files changed, 385 insertions(+), 166 deletions(-) create mode 100644 inttest/inttest_utils.erl diff --git a/appveyor.yml b/appveyor.yml index a13f3d61..59b9570c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,13 +1,24 @@ version: 2.6.1.{build} -branches: - only: - - master +clone_depth: 1 build_script: -- cmd: bootstrap.bat +- cmd: >- + bootstrap.bat + + set REBAR_EXTRA_DEPS=1 + + .\rebar get-deps + + copy rebar deps\retest + + copy rebar.cmd deps\retest + + cd deps\retest & .\rebar compile escriptize test_script: - cmd: >- rebar eunit - deps/retest/retest -l debug inttest + "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 + + cd deps\retest & retest -l debug -t 120000 ..\..\inttest artifacts: - path: rebar diff --git a/inttest/app_src/app_src_rt.erl b/inttest/app_src/app_src_rt.erl index d71f03ee..2b5a87e9 100644 --- a/inttest/app_src/app_src_rt.erl +++ b/inttest/app_src/app_src_rt.erl @@ -30,9 +30,14 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, - {create, "src/app_src.app.src", app(app_src)}]. + [ + {create, "src/app_src.app.src", app(app_src)} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/app_src_script/app_src_script_rt.erl b/inttest/app_src_script/app_src_script_rt.erl index 4c00ec87..c0de128e 100644 --- a/inttest/app_src_script/app_src_script_rt.erl +++ b/inttest/app_src_script/app_src_script_rt.erl @@ -30,9 +30,14 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, - {create, "src/app_src_script.app.src.script", app_script(app_src_script)}]. + [ + {create, "src/app_src_script.app.src.script", app_script(app_src_script)} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/app_src_script_2/app_src_script_2_rt.erl b/inttest/app_src_script_2/app_src_script_2_rt.erl index 1bf59d13..613e3f9e 100644 --- a/inttest/app_src_script_2/app_src_script_2_rt.erl +++ b/inttest/app_src_script_2/app_src_script_2_rt.erl @@ -30,10 +30,15 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, + [ {create, "src/app_src_script_2.app.src.script", app_script(app_src_script_2)}, - {create, "src/app_src_script_2.app.src", app(app_src_script_2)}]. + {create, "src/app_src_script_2.app.src", app(app_src_script_2)} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/appup_src/appup_src_rt.erl b/inttest/appup_src/appup_src_rt.erl index 2ca6788a..7530aa49 100644 --- a/inttest/appup_src/appup_src_rt.erl +++ b/inttest/appup_src/appup_src_rt.erl @@ -30,9 +30,14 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, - {copy, "src", "src"}]. + [ + {copy, "src", "src"} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/appup_src_2/appup_src_2_rt.erl b/inttest/appup_src_2/appup_src_2_rt.erl index 09d25039..99e62487 100644 --- a/inttest/appup_src_2/appup_src_2_rt.erl +++ b/inttest/appup_src_2/appup_src_2_rt.erl @@ -30,9 +30,14 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, - {copy, "src", "src"}]. + [ + {copy, "src", "src"} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/appup_src_2/appup_src_rt_2.erl b/inttest/appup_src_2/appup_src_rt_2.erl index d662183d..fff6f627 100644 --- a/inttest/appup_src_2/appup_src_rt_2.erl +++ b/inttest/appup_src_2/appup_src_rt_2.erl @@ -30,9 +30,13 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, - {copy, "src", "src"}]. + [{copy, "src", "src"} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/appup_src_script/appup_src_script_rt.erl b/inttest/appup_src_script/appup_src_script_rt.erl index c98e54ea..1aeb680a 100644 --- a/inttest/appup_src_script/appup_src_script_rt.erl +++ b/inttest/appup_src_script/appup_src_script_rt.erl @@ -30,9 +30,14 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, - {copy, "src", "src"}]. + [ + {copy, "src", "src"} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/bug_5_rt.erl b/inttest/bug_5_rt.erl index 8894cb57..f198d40d 100644 --- a/inttest/bug_5_rt.erl +++ b/inttest/bug_5_rt.erl @@ -4,20 +4,21 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. files() -> [{create, "ebin/a1.app", app(a1)}, {create, "deps/d1/src/d1.app.src", app(d1)}, {create, "rebar.config", - <<"{deps, [{d1, \"1\", {hg, \"http://example.com\", \"tip\"}}]}.\n">>}, - {copy, "../rebar", "rebar"}]. + <<"{deps, [{d1, \"1\", {hg, \"http://example.com\", \"tip\"}}]}.\n">>} + ] ++ inttest_utils:rebar_setup(".."). run(_Dir) -> {ok, _} = retest:sh("./rebar compile"), ok. - - %% %% Generate the contents of a simple .app file %% diff --git a/inttest/code_path_no_recurse/code_path_no_recurse_rt.erl b/inttest/code_path_no_recurse/code_path_no_recurse_rt.erl index 74d035e2..a2ca9806 100644 --- a/inttest/code_path_no_recurse/code_path_no_recurse_rt.erl +++ b/inttest/code_path_no_recurse/code_path_no_recurse_rt.erl @@ -28,14 +28,17 @@ -export([files/0, run/1]). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "src", "src"}, {copy, "test", "test"}, {copy, "deps", "deps"} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> retest:log(info, "Compile project~n"), diff --git a/inttest/cover/cover_rt.erl b/inttest/cover/cover_rt.erl index a9f3f08b..3f5e4853 100644 --- a/inttest/cover/cover_rt.erl +++ b/inttest/cover/cover_rt.erl @@ -30,13 +30,18 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{create, "ebin/foo.app", app(foo)}, - {copy, "../../rebar","rebar"}, + [ + {create, "ebin/foo.app", app(foo)}, {copy, "src", "src"}, {copy, "rebar-cover_export_json.config", - "rebar-cover_export_json.config"}]. + "rebar-cover_export_json.config"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> ifdef_test(), diff --git a/inttest/ct1/ct1_rt.erl b/inttest/ct1/ct1_rt.erl index dc830955..03b4ed8d 100644 --- a/inttest/ct1/ct1_rt.erl +++ b/inttest/ct1/ct1_rt.erl @@ -4,21 +4,23 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. files() -> - [{create, "ebin/a1.app", app(a1)}, - {copy, "../../rebar", "rebar"}, + [ + {create, "ebin/a1.app", app(a1)}, {copy, "rebar.config", "rebar.config"}, {copy, "app.config", "app.config"}, - {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}]. + {copy, "test_SUITE.erl", "itest/test_SUITE.erl"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> {ok, _} = retest:sh("./rebar compile ct"), {ok, _} = retest:sh("./rebar compile ct -v"), ok. - - %% %% Generate the contents of a simple .app file %% diff --git a/inttest/ct2/ct2_rt.erl b/inttest/ct2/ct2_rt.erl index f9d2b199..b3138d40 100644 --- a/inttest/ct2/ct2_rt.erl +++ b/inttest/ct2/ct2_rt.erl @@ -4,13 +4,17 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. files() -> - [{create, "ebin/foo.app", app(foo)}, - {copy, "../../rebar", "rebar"}, + [ + {create, "ebin/foo.app", app(foo)}, {copy, "foo.test.spec", "foo.test.spec"}, {copy, "deps/bar.test.spec", "deps/bar.test.spec"}, - {copy, "foo_SUITE.erl", "test/foo_SUITE.erl"}]. + {copy, "foo_SUITE.erl", "test/foo_SUITE.erl"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> Ref = retest:sh("./rebar compile ct -vvv", [async]), diff --git a/inttest/ct3/ct3_rt.erl b/inttest/ct3/ct3_rt.erl index a87cf210..b5739db5 100644 --- a/inttest/ct3/ct3_rt.erl +++ b/inttest/ct3/ct3_rt.erl @@ -28,19 +28,23 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. files() -> - [{create, "ebin/a1.app", app(a1)}, - {copy, "../../rebar", "rebar"}, + [ + {create, "ebin/a1.app", app(a1)}, {copy, "rebar.config", "rebar.config"}, {copy, "app.config", "itest/app.config"}, {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}, {copy, "converted"}, - {copy, "unconverted"}]. + {copy, "unconverted"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> {ok, _} = retest:sh("./rebar compile ct -v", - [{env, [{"ERL_FLAGS", "-name ct_rt3"}]}]), + [{env, [{"ERL_FLAGS", "-name ct_rt3@localhost"}]}]), ok. %% diff --git a/inttest/ct_cover/ct_cover_rt.erl b/inttest/ct_cover/ct_cover_rt.erl index c7f8776f..361d85bd 100644 --- a/inttest/ct_cover/ct_cover_rt.erl +++ b/inttest/ct_cover/ct_cover_rt.erl @@ -4,14 +4,19 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{create, "ebin/a1.app", app(a1)}, - {copy, "../../rebar", "rebar"}, + [ + {create, "ebin/a1.app", app(a1)}, {copy, "rebar.config", "rebar.config"}, {copy, "app.config", "app.config"}, {copy, "cover.spec", "cover.spec"}, {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}, - {copy, "mock", "deps"}]. + {copy, "mock", "deps"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> {ok, _} = retest:sh("./rebar compile ct"), diff --git a/inttest/depplugins/depplugins_rt.erl b/inttest/depplugins/depplugins_rt.erl index 7bcfe86e..fc72dec2 100644 --- a/inttest/depplugins/depplugins_rt.erl +++ b/inttest/depplugins/depplugins_rt.erl @@ -21,9 +21,12 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "base_dir_cwd_plugin.erl", "base_dir_cwd_plugin.erl"}, {create, "ebin/fish.app", app(fish, [])}, @@ -38,7 +41,7 @@ files() -> "deps/testplugin/plugins/testplugin_mod.erl"}, {copy, "dep_cwd_plugin.erl", "deps/testplugin/dep_cwd_plugin.erl"}, {create, "deps/testplugin/ebin/testplugin.app", app(testplugin, [])} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), diff --git a/inttest/erlc/erlc_rt.erl b/inttest/erlc/erlc_rt.erl index 0c1f25d3..3ac59560 100644 --- a/inttest/erlc/erlc_rt.erl +++ b/inttest/erlc/erlc_rt.erl @@ -54,9 +54,12 @@ "foo_worker.beam", "SIMPLE-ASN.beam"]). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "rebar-no_debug_info.config", "rebar-no_debug_info.config"}, {copy, "include", "include"}, @@ -73,7 +76,7 @@ files() -> %% deps {create, "deps/foobar/ebin/foobar.app", app(foobar, [foobar])}, {copy, "foobar.erl", "deps/foobar/src/foobar.erl"} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), diff --git a/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl b/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl index 384ce87a..72c69289 100644 --- a/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl +++ b/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl @@ -30,39 +30,57 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, + [ {copy, "rebar.config", "rebar.config"}, {copy, "src", "src"}, {copy, "include", "include"}, - {copy, "extra_include", "extra_include"}]. + {copy, "extra_include", "extra_include"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> + retest_log:log(debug, "compiling all...\n\n", []), compile_all(ok, ""), + retest_log:log(debug, "checking beams integrity...\n\n", []), check_beams_ok(), check_beams_untouched(filelib:wildcard("ebin/*.beam")), + retest_log:log(debug, "modifying lisp.erl and recompiling...\n\n", []), modify_and_recompile_ok("src/lisp.erl", "ebin/lisp.beam"), + retest_log:log(debug, "cleaning all...\n\n", []), clean_all_ok(), + retest_log:log(debug, "compiling all (expect fail)...\n\n", []), compile_all(error, "-C rebar.config.non-existing"), + retest_log:log(debug, "compiling all...\n\n", []), compile_all(ok, ""), + retest_log:log(debug, "modifying extra_include/extra.hrl and recompiling...\n\n", []), modify_and_recompile_ok("extra_include/extra.hrl", "ebin/java.beam"), + retest_log:log(debug, "rewriting src/java.erl...\n\n", []), Java = "src/java.erl", {ok, OrigContent} = file:read_file(Java), %% Remove header file inclusion {ok, _} = file:copy("src/java.erl.no_extra", Java), %% Ensure recompilation touch([Java]), + retest_log:log(debug, "compiling all...\n\n", []), compile_all(ok, ""), %% Modify that header file + retest_log:log(debug, "again modifying extra_include/extra.hrl and recompiling...\n\n", []), touch(["extra_include/extra.hrl"]), %% Ensure we don't have to recompile anything + retest_log:log(debug, "ensure ebin/java.beam was untouched...\n\n", []), check_beams_untouched(["ebin/java.beam"]), %% Clean up + retest_log:log(debug, "modifying src/java.erl...\n\n", []), ok = file:write_file(Java, OrigContent), %% Check that changes propagate deeply through the dependency tree + retest_log:log(debug, "modifying include/lambda.hrl...\n\n", []), modify_and_recompile_ok("include/lambda.hrl", "ebin/perl.beam"), ok. @@ -82,7 +100,9 @@ compile_all_and_assert_mtimes(Beams, Cmp) -> BeamsModifiedBefore = mtime_ns(Beams), compile_all(ok, ""), BeamsModifiedAfter = mtime_ns(Beams), - lists:zipwith(fun(Before, After) -> ?assert(Cmp(Before, After)) end, + lists:zipwith(fun(Before, After) -> + ?assert(Cmp(Before, After)) + end, BeamsModifiedBefore, BeamsModifiedAfter). with_erl_beams(F) -> @@ -95,13 +115,13 @@ with_erl_beams(F) -> filelib:wildcard("src/*.erl")). mtime_ns(Files) -> - [os:cmd("stat -c%y " ++ File) || File <- Files]. + [calendar:datetime_to_gregorian_seconds(filelib:last_modified(File)) || File <- Files]. touch(Files) -> %% Sleep one second so that filelib:last_modified/1 is guaranteed to notice %% that files have changed. ok = timer:sleep(1000), - [os:cmd("touch " ++ File) || File <- Files]. + [file:change_time(File, calendar:local_time()) || File <- Files]. compile_all(Result, Opts) -> ?assertMatch({Result, _}, diff --git a/inttest/eunit/eunit_rt.erl b/inttest/eunit/eunit_rt.erl index 1d714049..bc24b26a 100644 --- a/inttest/eunit/eunit_rt.erl +++ b/inttest/eunit/eunit_rt.erl @@ -5,14 +5,19 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{create, "ebin/foo.app", app(foo)}, - {copy, "../../rebar", "rebar"}, + [ + {create, "ebin/foo.app", app(foo)}, {copy, "src", "src"}, {copy, "eunit_src", "eunit_src"}, {copy, "rebar-eunit_compile_opts.config", - "rebar-eunit_compile_opts.config"}]. + "rebar-eunit_compile_opts.config"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> ifdef_test(), diff --git a/inttest/inttest_utils.erl b/inttest/inttest_utils.erl new file mode 100644 index 00000000..d896f4d3 --- /dev/null +++ b/inttest/inttest_utils.erl @@ -0,0 +1,16 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(inttest_utils). + +-compile(export_all). + +rebar_setup({win32, nt}, Dir) -> + [{copy, filename:join(Dir, "rebar.cmd"), "rebar.cmd"}]; +rebar_setup({_, _}, _) -> []. + +rebar_setup(Dir) -> + [{copy, + filename:join(Dir, "rebar"), "rebar"}] ++ rebar_setup(os:type(), Dir). + +rebar_setup() -> + rebar_setup("../.."). diff --git a/inttest/logging/logging_rt.erl b/inttest/logging/logging_rt.erl index 2709a84e..f7743765 100644 --- a/inttest/logging/logging_rt.erl +++ b/inttest/logging/logging_rt.erl @@ -30,11 +30,14 @@ -define(APP_FILE, "ebin/logging.app"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {create, ?APP_FILE, app(invalid_name, [])} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> SharedExpected = "==> logging_rt \\(compile\\)", diff --git a/inttest/port/port_rt.erl b/inttest/port/port_rt.erl index 90ecbdcf..c910e0ed 100644 --- a/inttest/port/port_rt.erl +++ b/inttest/port/port_rt.erl @@ -31,13 +31,16 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "c_src", "c_src"}, {create, "ebin/foo.app", app(foo, [])} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> %% wait a bit for new files to have different timestamps @@ -45,48 +48,76 @@ run(_Dir) -> %% test.so is created during first compile ?assertEqual(0, filelib:last_modified("priv/test.so")), ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), - TestSo1 = filelib:last_modified("priv/test.so"), + TestSo1 = filelib:last_modified("priv/test" ++ + shared_library_file_extension(os:type())), ?assert(TestSo1 > 0), wait(), %% nothing happens during second compile ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), - TestSo2 = filelib:last_modified("priv/test.so"), - Test1o2 = filelib:last_modified("c_src/test1.o"), - Test2o2 = filelib:last_modified("c_src/test2.o"), + TestSo2 = filelib:last_modified("priv/test" ++ + shared_library_file_extension(os:type())), + Test1o2 = filelib:last_modified("c_src/test1" ++ + object_file_extension(os:type())), + Test2o2 = filelib:last_modified("c_src/test2" ++ + object_file_extension(os:type())), ?assertEqual(TestSo1, TestSo2), ?assert(TestSo1 >= Test1o2), ?assert(TestSo1 >= Test2o2), wait(), %% when test2.c changes, at least test2.o and test.so are rebuilt - ?assertMatch({ok, _}, retest_sh:run("touch c_src/test2.c", [])), + ?assertMatch({ok, _}, retest:run({touch, "c_src/test2.c"}, [{dir, "."}])), ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), - TestSo3 = filelib:last_modified("priv/test.so"), - Test2o3 = filelib:last_modified("c_src/test2.o"), + TestSo3 = filelib:last_modified("priv/test" ++ + shared_library_file_extension(os:type())), + Test2o3 = filelib:last_modified("c_src/test2" ++ + object_file_extension(os:type())), ?assert(TestSo3 > TestSo2), ?assert(Test2o3 > TestSo2), - wait(), - %% when test2.h changes, at least test2.o and test.so are rebuilt - ?assertMatch({ok, _}, retest_sh:run("touch c_src/test2.h", [])), - ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), - TestSo4 = filelib:last_modified("priv/test.so"), - Test2o4 = filelib:last_modified("c_src/test2.o"), - ?assert(TestSo4 > TestSo3), - ?assert(Test2o4 > TestSo3), - wait(), - %% when test1.h changes, everything is rebuilt - ?assertMatch({ok, _}, retest_sh:run("touch c_src/test1.h", [])), - ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), - TestSo5 = filelib:last_modified("priv/test.so"), - Test1o5 = filelib:last_modified("c_src/test1.o"), - Test2o5 = filelib:last_modified("c_src/test2.o"), - ?assert(TestSo5 > TestSo4), - ?assert(Test1o5 > TestSo4), - ?assert(Test2o5 > TestSo4), - ok. + %% detecting the a full recompile is needed when changing a .h file is a feature attained + %% by using the -MMD gcc flag which sadly is not available in Windows, so this part of the + %% test is only executed in Unix + case os:type() of + {win32, _} -> ok; + _ -> + wait(), + %% when test2.h changes, at least test2.o and test.so are rebuilt + ?assertMatch({ok, _}, + retest:run({touch, "c_src/test2.h"}, [{dir, "."}])), + ?assertMatch({ok, _}, + retest_sh:run("./rebar compile", [])), + TestSo4 = filelib:last_modified("priv/test" ++ + shared_library_file_extension(os:type())), + Test2o4 = filelib:last_modified("c_src/test2" ++ + object_file_extension(os:type())), + ?assert(TestSo4 > TestSo3), + ?assert(Test2o4 > TestSo3), + wait(), + %% when test1.h changes, everything is rebuilt + ?assertMatch({ok, _}, + retest:run({touch, "c_src/test1.h"}, [{dir, "."}])), + ?assertMatch({ok, _}, + retest_sh:run("./rebar compile", [])), + TestSo5 = filelib:last_modified("priv/test" ++ + shared_library_file_extension(os:type())), + Test1o5 = filelib:last_modified("c_src/test1" ++ + object_file_extension(os:type())), + Test2o5 = filelib:last_modified("c_src/test2" ++ + object_file_extension(os:type())), + ?assert(TestSo5 > TestSo4), + ?assert(Test1o5 > TestSo4), + ?assert(Test2o5 > TestSo4), + ok + end. wait() -> timer:sleep(1000). +object_file_extension({win32, nt}) -> ".o"; +object_file_extension(_) -> ".o". + +shared_library_file_extension({win32, nt}) -> ".dll"; +shared_library_file_extension(_) -> ".so". + %% %% Generate the contents of a simple .app file %% diff --git a/inttest/port/rebar.config b/inttest/port/rebar.config index a9412187..efab1aff 100644 --- a/inttest/port/rebar.config +++ b/inttest/port/rebar.config @@ -1 +1,2 @@ -{port_specs, [{"priv/test.so", ["c_src/*.c"]}]}. +{port_specs, [{"win32", "priv/test.dll", ["c_src/*.c"]}, + {"priv/test.so", ["c_src/*.c"]}]}. diff --git a/inttest/profile/profile_rt.erl b/inttest/profile/profile_rt.erl index b1285176..b8a9e09b 100644 --- a/inttest/profile/profile_rt.erl +++ b/inttest/profile/profile_rt.erl @@ -30,10 +30,12 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [ - {copy, "../../rebar", "rebar"} - ]. + inttest_utils:rebar_setup(). run(_Dir) -> Cmd = "./rebar list-deps", diff --git a/inttest/proto_gpb/proto_gpb_rt.erl b/inttest/proto_gpb/proto_gpb_rt.erl index 8a7cacf8..307aca0c 100644 --- a/inttest/proto_gpb/proto_gpb_rt.erl +++ b/inttest/proto_gpb/proto_gpb_rt.erl @@ -30,7 +30,6 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("kernel/include/file.hrl"). --include_lib("deps/retest/include/retest.hrl"). -define(MODULES, [foo, @@ -51,9 +50,12 @@ "c/test4.proto", "c/d/test5.proto"]). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "rebar2.config", "rebar2.config"}, {copy, "rebar.bad.config", "rebar.bad.config"}, @@ -63,7 +65,7 @@ files() -> {copy, "proto.bad", "proto.bad"}, {copy, "mock", "deps"}, {create, "ebin/foo.app", app(foo, ?MODULES ++ ?GENERATED_MODULES)} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> % perform test obtaining the .proto files from src dir @@ -99,7 +101,7 @@ run_from_dir(success_expected, ProtoDir, ConfigFile) -> %% the .hrl file was generated before foo was compiled. ok = check_beams_generated(), - ?DEBUG("Verifying recompilation~n", []), + retest_log:log(debug, "Verifying recompilation~n", []), TestErl = hd(generated_erl_files()), TestProto = hd(source_proto_files(ProtoDir)), make_proto_newer_than_erl(TestProto, TestErl), @@ -111,7 +113,7 @@ run_from_dir(success_expected, ProtoDir, ConfigFile) -> TestMTime2 = read_mtime(TestErl), ?assert(TestMTime2 > TestMTime1), - ?DEBUG("Verifying recompilation with no changes~n", []), + retest_log:log(debug, "Verifying recompilation with no changes~n", []), TestMTime3 = read_mtime(TestErl), ?assertMatch({ok, _}, retest_sh:run("./rebar --config " ++ ConfigFile @@ -120,7 +122,7 @@ run_from_dir(success_expected, ProtoDir, ConfigFile) -> TestMTime4 = read_mtime(TestErl), ?assert(TestMTime3 =:= TestMTime4), - ?DEBUG("Verify cleanup~n", []), + retest_log:log(debug, "Verify cleanup~n", []), ?assertMatch({ok, _}, retest_sh:run("./rebar --config " ++ ConfigFile ++ " clean", diff --git a/inttest/proto_protobuffs/proto_protobuffs_rt.erl b/inttest/proto_protobuffs/proto_protobuffs_rt.erl index 1bb7b5ec..ee4c294d 100644 --- a/inttest/proto_protobuffs/proto_protobuffs_rt.erl +++ b/inttest/proto_protobuffs/proto_protobuffs_rt.erl @@ -42,15 +42,18 @@ "foo_sup.beam", "test_pb.beam"]). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "include", "include"}, {copy, "src", "src"}, {copy, "mock", "deps"}, {create, "ebin/foo.app", app(foo, ?MODULES)} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> ?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])), diff --git a/inttest/require_vsn/require_vsn_rt.erl b/inttest/require_vsn/require_vsn_rt.erl index 0b947c5a..8ff87f76 100644 --- a/inttest/require_vsn/require_vsn_rt.erl +++ b/inttest/require_vsn/require_vsn_rt.erl @@ -28,12 +28,15 @@ -export([files/0, run/1]). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {create, "ebin/require_vsn.app", app(require_vsn, [])} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> SharedExpected = "==> require_vsn_rt \\(compile\\)", diff --git a/inttest/rgen1/retest.config b/inttest/rgen1/retest.config index b569f148..244096d4 100644 --- a/inttest/rgen1/retest.config +++ b/inttest/rgen1/retest.config @@ -1 +1 @@ -{timeout, 120000}. +{timeout, 240000}. diff --git a/inttest/rgen1/rgen1_rt.erl b/inttest/rgen1/rgen1_rt.erl index 1bf36c26..5847f4a1 100644 --- a/inttest/rgen1/rgen1_rt.erl +++ b/inttest/rgen1/rgen1_rt.erl @@ -6,13 +6,16 @@ %% Exercise release generation w/ templating +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ {copy, "reltool.config"}, {copy, "test.config"}, - {copy, "vars.config"}, - {copy, "../../rebar"} - ]. + {copy, "vars.config"} + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> {ok, _} = retest_sh:run("./rebar -v generate", []), diff --git a/inttest/t_custom_config/t_custom_config_rt.erl b/inttest/t_custom_config/t_custom_config_rt.erl index a373b327..1f21927c 100644 --- a/inttest/t_custom_config/t_custom_config_rt.erl +++ b/inttest/t_custom_config/t_custom_config_rt.erl @@ -6,10 +6,15 @@ -include_lib("eunit/include/eunit.hrl"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> - [{copy, "../../rebar", "rebar"}, + [ {copy, "custom.config", "custom.config"}, - {create, "ebin/custom_config.app", app(custom_config, [custom_config])}]. + {create, "ebin/custom_config.app", app(custom_config, [custom_config])} + ] ++ inttest_utils:rebar_setup(). run(Dir) -> retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), diff --git a/inttest/tdeps1/tdeps1_rt.erl b/inttest/tdeps1/tdeps1_rt.erl index 3591ec01..849403b5 100644 --- a/inttest/tdeps1/tdeps1_rt.erl +++ b/inttest/tdeps1/tdeps1_rt.erl @@ -4,16 +4,18 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + %% Exercise transitive dependencies %% A -> B -> C, where A includes a .hrl from B which includes .hrl from C - files() -> [ %% A application {create, "ebin/a.app", app(a, [a])}, {copy, "a.rebar.config", "rebar.config"}, {copy, "a.erl", "src/a.erl"}, - {copy, "../../rebar", "rebar"}, %% B application {create, "repo/b/ebin/b.app", app(b, [])}, @@ -23,7 +25,7 @@ files() -> %% C application {create, "repo/c/ebin/c.app", app(c, [])}, {copy, "c.hrl", "repo/c/include/c.hrl"} - ]. + ] ++ inttest_utils:rebar_setup(). apply_cmds([], _Params) -> ok; @@ -39,7 +41,7 @@ run(_Dir) -> "git add -A", "git config user.email 'tdeps@example.com'", "git config user.name 'tdeps'", - "git commit -a -m 'Initial Commit'"], + "git commit -a -m \"Initial Commit\""], apply_cmds(GitCmds, [{dir, "repo/b"}]), apply_cmds(GitCmds, [{dir, "repo/c"}]), diff --git a/inttest/tdeps2/tdeps2_rt.erl b/inttest/tdeps2/tdeps2_rt.erl index 97a24ce8..a5665ad2 100644 --- a/inttest/tdeps2/tdeps2_rt.erl +++ b/inttest/tdeps2/tdeps2_rt.erl @@ -4,10 +4,13 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + %% Exercise transitive dependencies where there are multiple files %% depending on the same set of deps %% [A1, A2] -> B -> C ; A1 and A2 includes B.hrl which includes C.hrl - files() -> [ %% A1 application @@ -21,7 +24,6 @@ files() -> {template, "a.erl", "apps/a2/src/a2.erl", dict:from_list([{module, a2}])}, {copy, "root.rebar.config", "rebar.config"}, - {copy, "../../rebar", "rebar"}, %% B application {create, "repo/b/ebin/b.app", app(b, [])}, @@ -31,7 +33,7 @@ files() -> %% C application {create, "repo/c/ebin/c.app", app(c, [])}, {copy, "c.hrl", "repo/c/include/c.hrl"} - ]. + ] ++ inttest_utils:rebar_setup(). apply_cmds([], _Params) -> ok; @@ -47,7 +49,7 @@ run(_Dir) -> "git add -A", "git config user.email 'tdeps@example.com'", "git config user.name 'tdeps'", - "git commit -a -m 'Initial Commit'"], + "git commit -a -m \"Initial Commit\""], ok = apply_cmds(GitCmds, [{dir, "repo/b"}]), ok = apply_cmds(GitCmds, [{dir, "repo/c"}]), diff --git a/inttest/tdeps3/tdeps3_rt.erl b/inttest/tdeps3/tdeps3_rt.erl index f56b3cae..bc47772e 100644 --- a/inttest/tdeps3/tdeps3_rt.erl +++ b/inttest/tdeps3/tdeps3_rt.erl @@ -4,12 +4,15 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + %% Exercise transitive dependencies where there are multiple files %% depending on the same set of deps as well as lib_dir directives %% A -> B -> C -> D -> E %% |--> G(via lib_dir) %% |--> F -> D -> E - files() -> [ %% A1 application @@ -17,7 +20,6 @@ files() -> {template, "a.erl", "src/a.erl", dict:from_list([{module, a}, {dep, b}])}, {copy, "a.rebar.config", "rebar.config"}, - {copy, "../../rebar", "rebar"}, %% B application {create, "repo/b/ebin/b.app", app(b, [b])}, @@ -52,7 +54,7 @@ files() -> {create, "repo/b/apps/g/ebin/g.app", app(g, [])}, {copy, "e.hrl", "repo/b/apps/g/include/g.hrl"} - ]. + ] ++ inttest_utils:rebar_setup(). apply_cmds([], _Params) -> ok; @@ -68,7 +70,7 @@ run(_Dir) -> "git add -A", "git config user.email 'tdeps@example.com'", "git config user.name 'tdeps'", - "git commit -a -m 'Initial Commit'"], + "git commit -a -m \"Initial Commit\""], ok = apply_cmds(GitCmds, [{dir, "repo/b"}]), ok = apply_cmds(GitCmds, [{dir, "repo/c"}]), ok = apply_cmds(GitCmds, [{dir, "repo/d"}]), diff --git a/inttest/tdeps_update/tdeps_update_rt.erl b/inttest/tdeps_update/tdeps_update_rt.erl index c53e253c..b4919c50 100644 --- a/inttest/tdeps_update/tdeps_update_rt.erl +++ b/inttest/tdeps_update/tdeps_update_rt.erl @@ -4,6 +4,10 @@ -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + %% Exercises update deps, with recursive dependency updates. %% Initially: %% A(v0.5) -> B(v0.2.3) -> C(v1.0) @@ -25,7 +29,6 @@ files() -> {template, "a.erl", "apps/a1/src/a1.erl", dict:from_list([{module, a1}])}, {copy, "root.rebar.config", "rebar.config"}, - {copy, "../../rebar", "rebar"}, %% B application {create, "repo/b/ebin/b.app", app(b, [], "0.2.3")}, @@ -64,57 +67,62 @@ files() -> {copy, "c.rebar.config", "c.rebar.config"}, {copy, "c2.rebar.config", "c2.rebar.config"}, {copy, "c3.rebar.config", "c3.rebar.config"} - ]. + ] ++ inttest_utils:rebar_setup(). apply_cmds([], _Params) -> ok; apply_cmds([Cmd | Rest], Params) -> - io:format("Running: ~s (~p)\n", [Cmd, Params]), - {ok, _} = retest_sh:run(Cmd, Params), + io:format("Running: ~p (~p)\n", [Cmd, Params]), + {ok, _} = retest:run(Cmd, Params), apply_cmds(Rest, Params). -run(_Dir) -> +run(Dir) -> %% Initialize the b/c/d apps as git repos so that dependencies pull %% properly GitCmds = ["git init", "git add -A", - "git config user.email 'tdeps@example.com'", - "git config user.name 'tdeps'", - "git commit -a -m 'Initial Commit'"], + "git config user.email \"tdeps@example.com\"", + "git config user.name \"tdeps\"", + "git commit -a -m \"Initial Commit\""], BCmds = ["git tag 0.2.3", - "cp ../../b2.rebar.config rebar.config", - "cp ../../b2.app ebin/b.app", - "git commit -a -m 'update to 0.2.4'", + {copy, "../../b2.rebar.config", "rebar.config"}, + {copy, "../../b2.app", "ebin/b.app"}, + {touch, "ebin/b.app"}, + "git commit -a -m \"update to 0.2.4\"", "git tag 0.2.4", - "cp ../../b3.rebar.config rebar.config", - "cp ../../b3.app ebin/b.app", - "git commit -a -m 'update to 0.2.5'", + {copy, "../../b3.rebar.config", "rebar.config"}, + {copy, "../../b3.app", "ebin/b.app"}, + {touch, "ebin/b.app"}, + "git commit -a -m \"update to 0.2.5\"", "git tag 0.2.5", - "cp ../../b4.rebar.config rebar.config", - "cp ../../b4.app ebin/b.app", - "git commit -a -m 'update to 0.2.6'", + {copy, "../../b4.rebar.config", "rebar.config"}, + {copy, "../../b4.app", "ebin/b.app"}, + {touch, "ebin/b.app"}, + "git commit -a -m \"update to 0.2.6\"", "git tag 0.2.6"], %"git checkout 0.2.3"], CCmds = ["git tag 1.0", - "cp ../../c2.hrl include/c.hrl", - "cp ../../c2.app ebin/c.app", - "cp ../../c.rebar.config rebar.config", + {copy, "../../c2.hrl", "include/c.hrl"}, + {copy, "../../c2.app", "ebin/c.app"}, + {copy, "../../c.rebar.config", "rebar.config"}, "git add rebar.config", - "git commit -a -m 'update to 1.1'", + "git commit -a -m \"update to 1.1\"", "git tag 1.1", - "cp ../../c3.app ebin/c.app", - "cp ../../c2.rebar.config rebar.config", - "git commit -a -m 'update to 1.2'", + {copy, "../../c3.app", "ebin/c.app"}, + {copy, "../../c2.rebar.config", "rebar.config"}, + "git commit -a -m \"update to 1.2\"", "git tag 1.2", - "cp ../../c4.app ebin/c.app", - "cp ../../c3.rebar.config rebar.config", - "git commit -a -m 'update to 1.3'", + {copy, "../../c4.app", "ebin/c.app"}, + {copy, "../../c3.rebar.config", "rebar.config"}, + {touch, "rebar.config"}, + "git commit -a -m \"update to 1.3\"", "git tag 1.3"], %"git checkout 1.0"], DCmds = ["git tag 0.7"], ECmds = ["git tag 2.0", - "cp ../../e2.app ebin/e.app", - "git commit -a -m 'update to 2.1'", + {copy, "../../e2.app", "ebin/e.app"}, + {touch, "ebin/e.app"}, + "git commit -a -m \"update to 2.1\"", "git tag 2.1"], FCmds = ["git tag 0.1"], @@ -126,13 +134,16 @@ run(_Dir) -> {ok, _} = retest_sh:run("./rebar -v get-deps", []), {ok, _} = retest_sh:run("./rebar -v compile", []), - os:cmd("cp a2.rebar.config apps/a1/rebar.config"), + retest:run({copy, "a2.rebar.config", "apps/a1/rebar.config"}, + [{dir, Dir}]), {ok, _} = retest_sh:run("./rebar -v update-deps", []), {ok, _} = retest_sh:run("./rebar -v compile", []), - os:cmd("cp a3.rebar.config apps/a1/rebar.config"), + retest:run({copy, "a3.rebar.config", "apps/a1/rebar.config"}, + [{dir, Dir}]), {ok, _} = retest_sh:run("./rebar -v update-deps", []), {ok, _} = retest_sh:run("./rebar -v compile", []), - os:cmd("cp a4.rebar.config apps/a1/rebar.config"), + retest:run({copy, "a4.rebar.config", "apps/a1/rebar.config"}, + [{dir, Dir}]), {ok, _} = retest_sh:run("./rebar -v update-deps", []), {ok, _} = retest_sh:run("./rebar -v compile", []), ok. diff --git a/inttest/thooks/thooks_rt.erl b/inttest/thooks/thooks_rt.erl index 3cca6bbc..f1e7fa48 100644 --- a/inttest/thooks/thooks_rt.erl +++ b/inttest/thooks/thooks_rt.erl @@ -5,14 +5,17 @@ -include_lib("eunit/include/eunit.hrl"). -compile(export_all). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ %% dummy lfe files - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "fish.erl", "src/fish.erl"}, {create, "ebin/fish.app", app(fish, [fish])} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> ?assertMatch({ok, _}, retest_sh:run("./rebar -v clean compile", [])), @@ -27,7 +30,13 @@ ensure_command_ran_only_once(Command) -> ?assert(filelib:is_regular(File)), %% ensure that this command only ran once (not for each module) {ok, Content} = file:read_file(File), - ?assertEqual(Command ++ "\n", binary_to_list(Content)). + %% echo behaves differently in windows and unix + case os:type() of + {win32, nt} -> + ?assertEqual(Command ++ " \r\n", binary_to_list(Content)); + _ -> + ?assertEqual(Command ++ "\n", binary_to_list(Content)) + end. %% %% Generate the contents of a simple .app file diff --git a/inttest/tplugins/tplugins_rt.erl b/inttest/tplugins/tplugins_rt.erl index 01d296ea..d36afdd9 100644 --- a/inttest/tplugins/tplugins_rt.erl +++ b/inttest/tplugins/tplugins_rt.erl @@ -8,9 +8,12 @@ -define(COMPILE_ERROR, "ERROR: Plugin bad_plugin contains compilation errors:"). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "bad.config", "bad.config"}, {copy, "fish.erl", "src/fish.erl"}, @@ -18,7 +21,7 @@ files() -> {copy, "bad_plugin.erl", "bad_plugins/bad_plugin.erl"}, {create, "fwibble.test", <<"fwibble">>}, {create, "ebin/fish.app", app(fish, [fish])} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> ?assertMatch({ok, _}, retest_sh:run("./rebar fwibble -v", [])), diff --git a/inttest/xref_behavior/xref_behavior_rt.erl b/inttest/xref_behavior/xref_behavior_rt.erl index 8536710b..f6d5ff8b 100644 --- a/inttest/xref_behavior/xref_behavior_rt.erl +++ b/inttest/xref_behavior/xref_behavior_rt.erl @@ -2,16 +2,19 @@ -export([files/0, run/1]). +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + files() -> [ - {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, {copy, "xref_behavior.erl", "src/xref_behavior.erl"}, {copy, "gen_xref_behavior.erl", "src/gen_xref_behavior.erl"}, {create, "ebin/xref_behavior.app", app(xref_behavior, [xref_behavior, gen_xref_behavior])} - ]. + ] ++ inttest_utils:rebar_setup(). run(_Dir) -> {ok, _} = retest_sh:run("./rebar compile", []), diff --git a/rebar.config.script b/rebar.config.script index 25e3f807..78ba967d 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,7 +1,8 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 ft=erlang et -ExtraDeps = [{retest, ".*", {git, "git://github.com/dizzyd/retest.git", {tag, "4590941a"}}}], +ExtraDeps = [{retest, ".*", {git, "git://github.com/dizzyd/retest.git", + {tag, "1.1.0"}}}], case os:getenv("REBAR_EXTRA_DEPS") of false -> diff --git a/src/rebar_ct.erl b/src/rebar_ct.erl index 022dfc41..cf2059ec 100644 --- a/src/rebar_ct.erl +++ b/src/rebar_ct.erl @@ -113,7 +113,12 @@ run_test(TestDir, LogDir, Config, _File) -> false -> " >> " ++ RawLog ++ " 2>&1"; true -> + case os:type() of + {win32, nt} -> + " >> " ++ RawLog ++ " 2>&1"; + _ -> " 2>&1 | tee -a " ++ RawLog + end end, ShOpts = [{env,[{"TESTDIR", TestDir}]}, return_on_error], @@ -155,14 +160,27 @@ failure_logger(Command, {Rc, Output}) -> check_fail_log(Config, RawLog, Command, Result) -> check_log(Config, RawLog, failure_logger(Command, Result)). -check_log(Config,RawLog,Fun) -> - {ok, Msg} = - rebar_utils:sh("grep -e \"TEST COMPLETE\" -e \"{error,make_failed}\" " - ++ RawLog, [{use_stdout, false}]), - MakeFailed = string:str(Msg, "{error,make_failed}") =/= 0, - RunFailed = string:str(Msg, ", 0 failed") =:= 0, +check_log(Config,RawLogFilename,Fun) -> + %% read the file and split into a list separated by newlines + {ok, RawLog} = file:read_file(RawLogFilename), + Msg = string:tokens(binary_to_list(RawLog), "\n"), + %% now filter out all the list entries that do not have test + %% completion strings + CompleteRuns = lists:filter(fun(M) -> + string:str(M, "TEST COMPLETE") =/= 0 + end, Msg), + MakeFailed = lists:filter(fun(M) -> + string:str(M, "{error,make_failed}") =/= 0 + end, Msg), + %% the run has failed if at least one of the tests failed + RunFailed = lists:foldl(fun(M, Acc) -> + %% the "0 failed" string must be present for + %% the test to be considered successful + TestFailed = string:str(M, "0 failed") =:= 0, + TestFailed orelse Acc + end, false, CompleteRuns), if - MakeFailed -> + MakeFailed =/= [] -> show_log(Config, RawLog), ?ERROR("Building tests failed\n",[]), ?FAIL; @@ -182,8 +200,7 @@ show_log(Config, RawLog) -> ?CONSOLE("Showing log\n", []), case rebar_log:is_verbose(Config) of false -> - {ok, Contents} = file:read_file(RawLog), - ?CONSOLE("~s", [Contents]); + ?CONSOLE("~s", [RawLog]); true -> ok end. From dfb0994d96bbdfef26e7eeff153e2a17e8a8b2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Fri, 15 Jan 2016 15:10:44 +0000 Subject: [PATCH 34/89] Add the latest OTP 18 version to the Travis build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9494ca9f..e4a3149c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ otp_release: - R14B04 - R14B03 - 17.0 + - 18.0 before_script: - hostname -f - cc -v From 7c42aab0aa5b6c7fd560ea6981344d69e2343c23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Sun, 24 Jan 2016 15:36:33 +0000 Subject: [PATCH 35/89] Update retest dependency url Should point to rebar org fork, upstream no longer being actively maintained by owner. --- rebar.config.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.script b/rebar.config.script index 78ba967d..eacd510b 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,7 +1,7 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 ft=erlang et -ExtraDeps = [{retest, ".*", {git, "git://github.com/dizzyd/retest.git", +ExtraDeps = [{retest, ".*", {git, "git://github.com/rebar/retest.git", {tag, "1.1.0"}}}], case os:getenv("REBAR_EXTRA_DEPS") of From f5bc6df43989e53e60536674bf3c0f212b8962b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Thu, 13 Aug 2015 22:11:52 +0100 Subject: [PATCH 36/89] Add neotoma regression test Integration regression test that checks for correct generation of .erl files from .peg ones and that they are cleaned up. --- .../mock/neotoma/priv/peg_includes.hrl | 253 +++++++ .../neotoma1/mock/neotoma/src/neotoma.app.src | 10 + inttest/neotoma1/mock/neotoma/src/neotoma.erl | 145 ++++ .../mock/neotoma/src/neotoma_parse.erl | 625 ++++++++++++++++++ inttest/neotoma1/neotoma_src_rt.erl | 81 +++ inttest/neotoma1/rebar.config | 15 + inttest/neotoma1/src/csv.peg | 31 + inttest/neotoma1/src/neotoma1.app.src | 6 + 8 files changed, 1166 insertions(+) create mode 100644 inttest/neotoma1/mock/neotoma/priv/peg_includes.hrl create mode 100644 inttest/neotoma1/mock/neotoma/src/neotoma.app.src create mode 100644 inttest/neotoma1/mock/neotoma/src/neotoma.erl create mode 100644 inttest/neotoma1/mock/neotoma/src/neotoma_parse.erl create mode 100644 inttest/neotoma1/neotoma_src_rt.erl create mode 100644 inttest/neotoma1/rebar.config create mode 100644 inttest/neotoma1/src/csv.peg create mode 100644 inttest/neotoma1/src/neotoma1.app.src diff --git a/inttest/neotoma1/mock/neotoma/priv/peg_includes.hrl b/inttest/neotoma1/mock/neotoma/priv/peg_includes.hrl new file mode 100644 index 00000000..183b98c5 --- /dev/null +++ b/inttest/neotoma1/mock/neotoma/priv/peg_includes.hrl @@ -0,0 +1,253 @@ +-file("peg_includes.hrl", 1). +-type index() :: {{line, pos_integer()}, {column, pos_integer()}}. +-type input() :: binary(). +-type parse_failure() :: {fail, term()}. +-type parse_success() :: {term(), input(), index()}. +-type parse_result() :: parse_failure() | parse_success(). +-type parse_fun() :: fun((input(), index()) -> parse_result()). +-type xform_fun() :: fun((input(), index()) -> term()). + +-spec p(input(), index(), atom(), parse_fun(), xform_fun()) -> parse_result(). +p(Inp, StartIndex, Name, ParseFun, TransformFun) -> + case get_memo(StartIndex, Name) of % See if the current reduction is memoized + {ok, Memo} -> %Memo; % If it is, return the stored result + Memo; + _ -> % If not, attempt to parse + Result = case ParseFun(Inp, StartIndex) of + {fail,_} = Failure -> % If it fails, memoize the failure + Failure; + {Match, InpRem, NewIndex} -> % If it passes, transform and memoize the result. + Transformed = TransformFun(Match, StartIndex), + {Transformed, InpRem, NewIndex} + end, + memoize(StartIndex, Name, Result), + Result + end. + +-spec setup_memo() -> ets:tid(). +setup_memo() -> + put({parse_memo_table, ?MODULE}, ets:new(?MODULE, [set])). + +-spec release_memo() -> true. +release_memo() -> + ets:delete(memo_table_name()). + +-spec memoize(index(), atom(), parse_result()) -> true. +memoize(Index, Name, Result) -> + Memo = case ets:lookup(memo_table_name(), Index) of + [] -> []; + [{Index, Plist}] -> Plist + end, + ets:insert(memo_table_name(), {Index, [{Name, Result}|Memo]}). + +-spec get_memo(index(), atom()) -> {ok, term()} | {error, not_found}. +get_memo(Index, Name) -> + case ets:lookup(memo_table_name(), Index) of + [] -> {error, not_found}; + [{Index, Plist}] -> + case proplists:lookup(Name, Plist) of + {Name, Result} -> {ok, Result}; + _ -> {error, not_found} + end + end. + +-spec memo_table_name() -> ets:tid(). +memo_table_name() -> + get({parse_memo_table, ?MODULE}). + +-ifdef(p_eof). +-spec p_eof() -> parse_fun(). +p_eof() -> + fun(<<>>, Index) -> {eof, [], Index}; + (_, Index) -> {fail, {expected, eof, Index}} end. +-endif. + +-ifdef(p_optional). +-spec p_optional(parse_fun()) -> parse_fun(). +p_optional(P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} -> {[], Input, Index}; + {_, _, _} = Success -> Success + end + end. +-endif. + +-ifdef(p_not). +-spec p_not(parse_fun()) -> parse_fun(). +p_not(P) -> + fun(Input, Index)-> + case P(Input,Index) of + {fail,_} -> + {[], Input, Index}; + {Result, _, _} -> {fail, {expected, {no_match, Result},Index}} + end + end. +-endif. + +-ifdef(p_assert). +-spec p_assert(parse_fun()) -> parse_fun(). +p_assert(P) -> + fun(Input,Index) -> + case P(Input,Index) of + {fail,_} = Failure-> Failure; + _ -> {[], Input, Index} + end + end. +-endif. + +-ifdef(p_seq). +-spec p_seq([parse_fun()]) -> parse_fun(). +p_seq(P) -> + fun(Input, Index) -> + p_all(P, Input, Index, []) + end. + +-spec p_all([parse_fun()], input(), index(), [term()]) -> parse_result(). +p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index}; +p_all([P|Parsers], Inp, Index, Accum) -> + case P(Inp, Index) of + {fail, _} = Failure -> Failure; + {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum]) + end. +-endif. + +-ifdef(p_choose). +-spec p_choose([parse_fun()]) -> parse_fun(). +p_choose(Parsers) -> + fun(Input, Index) -> + p_attempt(Parsers, Input, Index, none) + end. + +-spec p_attempt([parse_fun()], input(), index(), none | parse_failure()) -> parse_result(). +p_attempt([], _Input, _Index, Failure) -> Failure; +p_attempt([P|Parsers], Input, Index, FirstFailure)-> + case P(Input, Index) of + {fail, _} = Failure -> + case FirstFailure of + none -> p_attempt(Parsers, Input, Index, Failure); + _ -> p_attempt(Parsers, Input, Index, FirstFailure) + end; + Result -> Result + end. +-endif. + +-ifdef(p_zero_or_more). +-spec p_zero_or_more(parse_fun()) -> parse_fun(). +p_zero_or_more(P) -> + fun(Input, Index) -> + p_scan(P, Input, Index, []) + end. +-endif. + +-ifdef(p_one_or_more). +-spec p_one_or_more(parse_fun()) -> parse_fun(). +p_one_or_more(P) -> + fun(Input, Index)-> + Result = p_scan(P, Input, Index, []), + case Result of + {[_|_], _, _} -> + Result; + _ -> + {fail, {expected, Failure, _}} = P(Input,Index), + {fail, {expected, {at_least_one, Failure}, Index}} + end + end. +-endif. + +-ifdef(p_label). +-spec p_label(atom(), parse_fun()) -> parse_fun(). +p_label(Tag, P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} = Failure -> + Failure; + {Result, InpRem, NewIndex} -> + {{Tag, Result}, InpRem, NewIndex} + end + end. +-endif. + +-ifdef(p_scan). +-spec p_scan(parse_fun(), input(), index(), [term()]) -> {[term()], input(), index()}. +p_scan(_, <<>>, Index, Accum) -> {lists:reverse(Accum), <<>>, Index}; +p_scan(P, Inp, Index, Accum) -> + case P(Inp, Index) of + {fail,_} -> {lists:reverse(Accum), Inp, Index}; + {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum]) + end. +-endif. + +-ifdef(p_string). +-spec p_string(binary()) -> parse_fun(). +p_string(S) -> + Length = erlang:byte_size(S), + fun(Input, Index) -> + try + <> = Input, + {S, Rest, p_advance_index(S, Index)} + catch + error:{badmatch,_} -> {fail, {expected, {string, S}, Index}} + end + end. +-endif. + +-ifdef(p_anything). +-spec p_anything() -> parse_fun(). +p_anything() -> + fun(<<>>, Index) -> {fail, {expected, any_character, Index}}; + (Input, Index) when is_binary(Input) -> + <> = Input, + {<>, Rest, p_advance_index(<>, Index)} + end. +-endif. + +-ifdef(p_charclass). +-spec p_charclass(string() | binary()) -> parse_fun(). +p_charclass(Class) -> + {ok, RE} = re:compile(Class, [unicode, dotall]), + fun(Inp, Index) -> + case re:run(Inp, RE, [anchored]) of + {match, [{0, Length}|_]} -> + {Head, Tail} = erlang:split_binary(Inp, Length), + {Head, Tail, p_advance_index(Head, Index)}; + _ -> {fail, {expected, {character_class, binary_to_list(Class)}, Index}} + end + end. +-endif. + +-ifdef(p_regexp). +-spec p_regexp(binary()) -> parse_fun(). +p_regexp(Regexp) -> + {ok, RE} = re:compile(Regexp, [unicode, dotall, anchored]), + fun(Inp, Index) -> + case re:run(Inp, RE) of + {match, [{0, Length}|_]} -> + {Head, Tail} = erlang:split_binary(Inp, Length), + {Head, Tail, p_advance_index(Head, Index)}; + _ -> {fail, {expected, {regexp, binary_to_list(Regexp)}, Index}} + end + end. +-endif. + +-ifdef(line). +-spec line(index() | term()) -> pos_integer() | undefined. +line({{line,L},_}) -> L; +line(_) -> undefined. +-endif. + +-ifdef(column). +-spec column(index() | term()) -> pos_integer() | undefined. +column({_,{column,C}}) -> C; +column(_) -> undefined. +-endif. + +-spec p_advance_index(input() | unicode:charlist() | pos_integer(), index()) -> index(). +p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings + lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput)); +p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters + {{line, Line}, {column, Col}} = Index, + case MatchedInput of + $\n -> {{line, Line+1}, {column, 1}}; + _ -> {{line, Line}, {column, Col+1}} + end. diff --git a/inttest/neotoma1/mock/neotoma/src/neotoma.app.src b/inttest/neotoma1/mock/neotoma/src/neotoma.app.src new file mode 100644 index 00000000..dff6cef4 --- /dev/null +++ b/inttest/neotoma1/mock/neotoma/src/neotoma.app.src @@ -0,0 +1,10 @@ +{application, neotoma, + [ + {description, "PEG/Packrat toolkit and parser-generator."}, + {vsn, "1.7.3"}, + {applications, [kernel, stdlib]}, + {contributors, ["Sean Cribbs"]}, + {licenses, ["MIT"]}, + {links, [{"Github", "https://github.com/seancribbs/neotoma"}]} + ] +}. diff --git a/inttest/neotoma1/mock/neotoma/src/neotoma.erl b/inttest/neotoma1/mock/neotoma/src/neotoma.erl new file mode 100644 index 00000000..f4102f60 --- /dev/null +++ b/inttest/neotoma1/mock/neotoma/src/neotoma.erl @@ -0,0 +1,145 @@ +-module(neotoma). +-author("Sean Cribbs "). +-export([file/1, file/2, bootstrap/0]). +-export([main/1]). + +-define(ALL_COMBINATORS, [p_eof, p_optional, p_not, p_assert, p_seq, + p_choose, p_zero_or_more, p_one_or_more, p_label, p_scan, + p_string, p_anything, p_charclass, p_regexp, line, column]). + +-type option() :: {module, atom()} | {output, file:filename()} | {transform_module, atom()} | + {neotoma_priv_dir, file:filename()}. + +%% @doc Handler function for escript. +-spec main(list()) -> ok | no_return(). +main([]) -> + io:format("Usage: neotoma filename [-module output_module] [-output output_dir] [-transform_module transform_module]\n"); +main([Filename | Args]) -> + %% code:priv_dir is unreliable when called in escript context, but + %% escript:script_name does what we want. + PrivDir = filename:join([filename:dirname(escript:script_name()), "priv"]), + file(Filename, [{neotoma_priv_dir, PrivDir} | parse_options(Args)]). + +%% @doc Generates a parser from the specified file. +%% @equiv file(Filename, []) +-spec file(file:filename()) -> ok | {error, atom()}. +file(InputGrammar) -> + file(InputGrammar, []). + +%% @doc Generates a parser from the specified file with the given options. +-spec file(file:filename(), [option()]) -> ok | {error, atom()}. +file(InputGrammar, Options) -> + Basename = filename:basename(InputGrammar, ".peg"), + InputDir = filename:dirname(InputGrammar), + ModuleName = proplists:get_value(module, Options, list_to_atom(Basename)), + OutputDir = proplists:get_value(output, Options, InputDir), + OutputFilename = filename:join(OutputDir, atom_to_list(ModuleName) ++ ".erl"), + TransformModule = proplists:get_value(transform_module, Options, false), + validate_params(filename:absname(InputGrammar), + ModuleName, + TransformModule, + filename:absname(OutputFilename)), + Parsed = parse_grammar(InputGrammar), + Rules = proplists:get_value(rules, Parsed), + Root = proplists:get_value(root, Parsed), + Code = proplists:get_value(code, Parsed), + GenTransform = proplists:get_value(transform, Parsed), + Combinators = proplists:get_value(combinators, Parsed, ?ALL_COMBINATORS), + ModuleAttrs = generate_module_attrs(ModuleName, Combinators), + EntryFuns = generate_entry_functions(Root), + TransformFun = create_transform(TransformModule, OutputDir, GenTransform), + PrivDir = proplists:get_value(neotoma_priv_dir, Options, code:priv_dir(neotoma)), + {ok, PegIncludes} = file:read_file(filename:join([PrivDir, "peg_includes.hrl"])), + file:write_file(OutputFilename, [ModuleAttrs, "\n", Code, "\n", EntryFuns, "\n", Rules, "\n", TransformFun, "\n", PegIncludes]). + +-spec validate_params(file:filename(),atom(),atom(),file:filename()) -> 'ok'. +validate_params(InputGrammar, _, _, OutputFile) when InputGrammar =:= OutputFile -> + throw({badarg, "Input and output file are the same!"}); +validate_params(_,_, false, _) -> ok; +validate_params(_,_, TransformModule, _) when not is_atom(TransformModule) -> + throw({badarg, "transform_module option must be an atom"}); +validate_params(_,Basename, TransformModule, _) when Basename =:= TransformModule -> + throw({badarg, "Transform module named same as parser module!"}); +validate_params(_,_, TransformModule, OutputFile) -> + OutMod = list_to_atom(filename:basename(OutputFile, ".erl")), + case OutMod of + TransformModule -> throw({badarg, "Transform module file same as parser output file!"}); + _ -> ok + end. + +-spec generate_module_attrs(atom(), [atom()]) -> iolist(). +generate_module_attrs(ModName, Combinators) -> + ["-module(", atom_to_list(ModName) ,").\n", + "-export([parse/1,file/1]).\n", + [ generate_combinator_macro(C) || Combinators /= undefined, C <- Combinators ], + "\n" + ]. + +generate_combinator_macro(C) -> + ["-define(", atom_to_list(C), ",true).\n"]. + +-spec generate_entry_functions({iodata(),_}) -> iolist(). +generate_entry_functions(Root) -> + {RootRule,_} = Root, + ["-spec file(file:name()) -> any().\n", + "file(Filename) -> case file:read_file(Filename) of {ok,Bin} -> parse(Bin); Err -> Err end.\n\n", + "-spec parse(binary() | list()) -> any().\n", + "parse(List) when is_list(List) -> parse(unicode:characters_to_binary(List));\n", + "parse(Input) when is_binary(Input) ->\n", + " _ = setup_memo(),\n", + " Result = case '",RootRule,"'(Input,{{line,1},{column,1}}) of\n", + " {AST, <<>>, _Index} -> AST;\n", + " Any -> Any\n" + " end,\n", + " release_memo(), Result.\n"]. + +-spec parse_grammar(file:filename()) -> any(). +parse_grammar(InputFile) -> + case neotoma_parse:file(InputFile) of + {fail, Index} -> + throw({grammar_error, {fail, Index}}); + {Parsed, Remainder, Index} -> + io:format("WARNING: Grammar parse ended unexpectedly at ~p, generated parser may be incorrect.~nRemainder:~n~p", + [Index, Remainder]), + Parsed; + L when is_list(L) -> L; + _ -> throw({error, {unknown, grammar, InputFile}}) + end. + +-spec create_transform(atom() | boolean(),file:filename(),_) -> iolist(). +create_transform(_,_,[]) -> []; +create_transform(false,_,_) -> + "transform(_,Node,_Index) -> Node."; +create_transform(ModName,Dir,_) when is_atom(ModName) -> + XfFile = filename:join(Dir, atom_to_list(ModName) ++ ".erl"), + case filelib:is_regular(XfFile) of + true -> io:format("'~s' already exists, skipping generation.~n", [XfFile]); + false -> generate_transform_stub(XfFile, ModName) + end, + ["transform(Symbol,Node,Index) -> ",atom_to_list(ModName),":transform(Symbol, Node, Index)."]. + +-spec generate_transform_stub(file:filename(), atom()) -> 'ok' | {'error',atom()}. +generate_transform_stub(XfFile,ModName) -> + Data = ["-module(",atom_to_list(ModName),").\n", + "-export([transform/3]).\n\n", + "%% Add clauses to this function to transform syntax nodes\n", + "%% from the parser into semantic output.\n", + "transform(Symbol, Node, _Index) when is_atom(Symbol) ->\n Node."], + file:write_file(XfFile, Data). + +%% @doc Bootstraps the neotoma metagrammar. Intended only for internal development! +%% @equiv file("src/neotoma_parse.peg") +-spec bootstrap() -> 'ok'. +bootstrap() -> + file("priv/neotoma_parse.peg", [{output, "src/"}, {neotoma_priv_dir, "priv"}]). + +%% @doc Parses arguments passed to escript +-spec parse_options(list()) -> list(). +parse_options(["-module", ModName | Rest]) -> + [{module, list_to_atom(ModName)} | parse_options(Rest)]; +parse_options(["-output", Dir | Rest]) -> + [{output, Dir} | parse_options(Rest)]; +parse_options(["-transform_module", ModName | Rest]) -> + [{transform_module, list_to_atom(ModName)} | parse_options(Rest)]; +parse_options([]) -> + []. diff --git a/inttest/neotoma1/mock/neotoma/src/neotoma_parse.erl b/inttest/neotoma1/mock/neotoma/src/neotoma_parse.erl new file mode 100644 index 00000000..1bffaf87 --- /dev/null +++ b/inttest/neotoma1/mock/neotoma/src/neotoma_parse.erl @@ -0,0 +1,625 @@ +-module(neotoma_parse). +-export([parse/1,file/1]). +-define(p_anything,true). +-define(p_charclass,true). +-define(p_choose,true). +-define(p_label,true). +-define(p_not,true). +-define(p_one_or_more,true). +-define(p_optional,true). +-define(p_scan,true). +-define(p_seq,true). +-define(p_string,true). +-define(p_zero_or_more,true). + + + +% insert escapes into a string +-spec escape_string(string()) -> string(). +escape_string(String) -> escape_string(String, []). + +-spec escape_string(string(), string()) -> string(). +escape_string([], Output) -> + lists:reverse(Output); +escape_string([H|T], Output) -> + escape_string(T, + case H of + $/ -> [$/,$\\|Output]; + $\" -> [$\",$\\|Output]; % " comment inserted to help some editors with highlighting the generated parser + $\' -> [$\',$\\|Output]; % ' comment inserted to help some editors with highlighting the generated parser + $\b -> [$b,$\\|Output]; + $\d -> [$d,$\\|Output]; + $\e -> [$e,$\\|Output]; + $\f -> [$f,$\\|Output]; + $\n -> [$n,$\\|Output]; + $\r -> [$r,$\\|Output]; + $\s -> [$s,$\\|Output]; + $\t -> [$t,$\\|Output]; + $\v -> [$v,$\\|Output]; + _ -> [H|Output] + end). + +-spec add_lhs(binary(), index()) -> true. +add_lhs(Symbol, Index) -> + case ets:lookup(memo_table_name(), lhs) of + [] -> + ets:insert(memo_table_name(), {lhs, [{Symbol,Index}]}); + [{lhs, L}] when is_list(L) -> + ets:insert(memo_table_name(), {lhs, [{Symbol,Index}|L]}) + end. + +-spec add_nt(binary(), index()) -> true | ok. +add_nt(Symbol, Index) -> + case ets:lookup(memo_table_name(), nts) of + [] -> + ets:insert(memo_table_name(), {nts, [{Symbol,Index}]}); + [{nts, L}] when is_list(L) -> + case proplists:is_defined(Symbol, L) of + true -> + ok; + _ -> + ets:insert(memo_table_name(), {nts, [{Symbol,Index}|L]}) + end + end. + +-spec verify_rules() -> ok | no_return(). +verify_rules() -> + [{lhs, LHS}] = ets:lookup(memo_table_name(), lhs), + [{nts, NTs}] = ets:lookup(memo_table_name(), nts), + [Root|NonRoots] = lists:reverse(LHS), + lists:foreach(fun({Sym,Idx}) -> + case proplists:is_defined(Sym, NTs) of + true -> + ok; + _ -> + io:format("neotoma warning: rule '~s' is unused. ~p~n", [Sym,Idx]) + end + end, NonRoots), + lists:foreach(fun({S,I}) -> + case proplists:is_defined(S, LHS) of + true -> + ok; + _ -> + io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]), + exit({neotoma, {no_reduction, list_to_atom(binary_to_list(S))}}) + end + end, NTs), + Root. + +-spec used_combinator(atom()) -> true. +used_combinator(C) -> + case ets:lookup(memo_table_name(), combinators) of + [] -> + ets:insert(memo_table_name(), {combinators, ordsets:from_list([C])}); + [{combinators, Cs}] -> + ets:insert(memo_table_name(), {combinators, ordsets:add_element(C, Cs)}) + end. + +-spec used_transform_variables(binary()) -> [ 'Node' | 'Idx' ]. +used_transform_variables(Transform) -> + Code = unicode:characters_to_list(Transform), + {ok, Tokens, _} = erl_scan:string(Code), + used_transform_variables(Tokens, []). + +used_transform_variables([{var, _, Name}|Tokens], Acc) -> + used_transform_variables(Tokens, case Name of + 'Node' -> [Name | Acc]; + 'Idx' -> [Name | Acc]; + _ -> Acc + end); +used_transform_variables([_|Tokens], Acc) -> + used_transform_variables(Tokens, Acc); +used_transform_variables([], Acc) -> + lists:usort(Acc). + +-spec file(file:name()) -> any(). +file(Filename) -> case file:read_file(Filename) of {ok,Bin} -> parse(Bin); Err -> Err end. + +-spec parse(binary() | list()) -> any(). +parse(List) when is_list(List) -> parse(unicode:characters_to_binary(List)); +parse(Input) when is_binary(Input) -> + _ = setup_memo(), + Result = case 'rules'(Input,{{line,1},{column,1}}) of + {AST, <<>>, _Index} -> AST; + Any -> Any + end, + release_memo(), Result. + +-spec 'rules'(input(), index()) -> parse_result(). +'rules'(Input, Index) -> + p(Input, Index, 'rules', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), fun 'declaration_sequence'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, _Idx) -> + RootRule = verify_rules(), + Rules = unicode:characters_to_binary(lists:map(fun(R) -> [R, "\n\n"] end, lists:nth(2, Node))), + Code = case lists:nth(4, Node) of + {code, Block} -> Block; + _ -> [] + end, + [{rules, Rules}, + {code, Code}, + {root, RootRule}, + {transform, ets:lookup(memo_table_name(),gen_transform)}, + {combinators, ets:lookup_element(memo_table_name(), combinators, 2)}] + + end). + +-spec 'declaration_sequence'(input(), index()) -> parse_result(). +'declaration_sequence'(Input, Index) -> + p(Input, Index, 'declaration_sequence', fun(I,D) -> (p_seq([p_label('head', fun 'declaration'/2), p_label('tail', p_zero_or_more(p_seq([fun 'space'/2, fun 'declaration'/2])))]))(I,D) end, fun(Node, _Idx) -> + FirstRule = proplists:get_value(head, Node), + OtherRules = [I || [_,I] <- proplists:get_value(tail, Node, [])], + [FirstRule|OtherRules] + end). + +-spec 'declaration'(input(), index()) -> parse_result(). +'declaration'(Input, Index) -> + p(Input, Index, 'declaration', fun(I,D) -> (p_seq([fun 'nonterminal'/2, p_zero_or_more(fun 'space'/2), p_string(<<"<-">>), p_zero_or_more(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2), p_string(<<";">>)]))(I,D) end, fun(Node, _Idx) -> + [{nonterminal,Symbol}|Tail] = Node, + add_lhs(Symbol, Index), + Transform = case lists:nth(6,Tail) of + {code, CodeBlock} -> CodeBlock; + _ -> + ets:insert_new(memo_table_name(),{gen_transform, true}), + ["transform('",Symbol,"', Node, Idx)"] + end, + TransformArgs = case used_transform_variables(Transform) of + [] -> "_Node, _Idx"; + ['Idx'] -> "_Node, Idx"; + ['Node'] -> "Node, _Idx"; + ['Idx', 'Node'] -> "Node, Idx" + end, + ["-spec '", Symbol, "'(input(), index()) -> parse_result().\n", + "'",Symbol,"'","(Input, Index) ->\n ", + "p(Input, Index, '",Symbol,"', fun(I,D) -> (", + lists:nth(4, Tail), + ")(I,D) end, fun(", TransformArgs, ") ->",Transform," end)."] + end). + +-spec 'parsing_expression'(input(), index()) -> parse_result(). +'parsing_expression'(Input, Index) -> + p(Input, Index, 'parsing_expression', fun(I,D) -> (p_choose([fun 'choice'/2, fun 'sequence'/2, fun 'primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'choice'(input(), index()) -> parse_result(). +'choice'(Input, Index) -> + p(Input, Index, 'choice', fun(I,D) -> (p_seq([p_label('head', fun 'alternative'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, p_string(<<"\/">>), fun 'space'/2, fun 'alternative'/2])))]))(I,D) end, fun(Node, _Idx) -> + Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)], + Head = proplists:get_value(head, Node), + Statements = [[", ", TS] || TS <- Tail], + used_combinator(p_choose), + ["p_choose([", Head, Statements, "])"] + end). + +-spec 'alternative'(input(), index()) -> parse_result(). +'alternative'(Input, Index) -> + p(Input, Index, 'alternative', fun(I,D) -> (p_choose([fun 'sequence'/2, fun 'labeled_primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'primary'(input(), index()) -> parse_result(). +'primary'(Input, Index) -> + p(Input, Index, 'primary', fun(I,D) -> (p_choose([p_seq([fun 'prefix'/2, fun 'atomic'/2]), p_seq([fun 'atomic'/2, fun 'suffix'/2]), fun 'atomic'/2]))(I,D) end, fun(Node, _Idx) -> +case Node of + [Atomic, one_or_more] -> + used_combinator(p_one_or_more), + used_combinator(p_scan), + ["p_one_or_more(", Atomic, ")"]; + [Atomic, zero_or_more] -> + used_combinator(p_zero_or_more), + used_combinator(p_scan), + ["p_zero_or_more(", Atomic, ")"]; + [Atomic, optional] -> + used_combinator(p_optional), + ["p_optional(", Atomic, ")"]; + [assert, Atomic] -> + used_combinator(p_assert), + ["p_assert(", Atomic, ")"]; + [not_, Atomic] -> + used_combinator(p_not), + ["p_not(", Atomic, ")"]; + _ -> Node +end + end). + +-spec 'sequence'(input(), index()) -> parse_result(). +'sequence'(Input, Index) -> + p(Input, Index, 'sequence', fun(I,D) -> (p_seq([p_label('head', fun 'labeled_primary'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, fun 'labeled_primary'/2])))]))(I,D) end, fun(Node, _Idx) -> + Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)], + Head = proplists:get_value(head, Node), + Statements = [[", ", TS] || TS <- Tail], + used_combinator(p_seq), + ["p_seq([", Head, Statements, "])"] + end). + +-spec 'labeled_primary'(input(), index()) -> parse_result(). +'labeled_primary'(Input, Index) -> + p(Input, Index, 'labeled_primary', fun(I,D) -> (p_seq([p_optional(fun 'label'/2), fun 'primary'/2]))(I,D) end, fun(Node, _Idx) -> + case hd(Node) of + [] -> lists:nth(2, Node); + Label -> + used_combinator(p_label), + ["p_label('", Label, "', ", lists:nth(2, Node), ")"] + end + end). + +-spec 'label'(input(), index()) -> parse_result(). +'label'(Input, Index) -> + p(Input, Index, 'label', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2), p_string(<<":">>)]))(I,D) end, fun(Node, _Idx) -> + lists:sublist(Node, length(Node)-1) + end). + +-spec 'suffix'(input(), index()) -> parse_result(). +'suffix'(Input, Index) -> + p(Input, Index, 'suffix', fun(I,D) -> (p_choose([fun 'repetition_suffix'/2, fun 'optional_suffix'/2]))(I,D) end, fun(Node, _Idx) -> + case Node of + <<"*">> -> zero_or_more; + <<"+">> -> one_or_more; + <<"?">> -> optional + end + end). + +-spec 'optional_suffix'(input(), index()) -> parse_result(). +'optional_suffix'(Input, Index) -> + p(Input, Index, 'optional_suffix', fun(I,D) -> (p_string(<<"?">>))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'repetition_suffix'(input(), index()) -> parse_result(). +'repetition_suffix'(Input, Index) -> + p(Input, Index, 'repetition_suffix', fun(I,D) -> (p_choose([p_string(<<"+">>), p_string(<<"*">>)]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'prefix'(input(), index()) -> parse_result(). +'prefix'(Input, Index) -> + p(Input, Index, 'prefix', fun(I,D) -> (p_choose([p_string(<<"&">>), p_string(<<"!">>)]))(I,D) end, fun(Node, _Idx) -> + case Node of + <<"&">> -> assert; + <<"!">> -> not_ + end + end). + +-spec 'atomic'(input(), index()) -> parse_result(). +'atomic'(Input, Index) -> + p(Input, Index, 'atomic', fun(I,D) -> (p_choose([fun 'terminal'/2, fun 'nonterminal'/2, fun 'parenthesized_expression'/2]))(I,D) end, fun(Node, _Idx) -> +case Node of + {nonterminal, Symbol} -> + [<<"fun '">>, Symbol, <<"'/2">>]; + _ -> Node +end + end). + +-spec 'parenthesized_expression'(input(), index()) -> parse_result(). +'parenthesized_expression'(Input, Index) -> + p(Input, Index, 'parenthesized_expression', fun(I,D) -> (p_seq([p_string(<<"(">>), p_optional(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_string(<<")">>)]))(I,D) end, fun(Node, _Idx) ->lists:nth(3, Node) end). + +-spec 'nonterminal'(input(), index()) -> parse_result(). +'nonterminal'(Input, Index) -> + p(Input, Index, 'nonterminal', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2)]))(I,D) end, fun(Node, Idx) -> + Symbol = unicode:characters_to_binary(Node), + add_nt(Symbol, Idx), + {nonterminal, Symbol} + end). + +-spec 'terminal'(input(), index()) -> parse_result(). +'terminal'(Input, Index) -> + p(Input, Index, 'terminal', fun(I,D) -> (p_choose([fun 'regexp_string'/2, fun 'quoted_string'/2, fun 'character_class'/2, fun 'anything_symbol'/2]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'regexp_string'(input(), index()) -> parse_result(). +'regexp_string'(Input, Index) -> + p(Input, Index, 'regexp_string', fun(I,D) -> (p_seq([p_string(<<"#">>), p_label('string', p_one_or_more(p_seq([p_not(p_string(<<"#">>)), p_choose([p_string(<<"\\#">>), p_anything()])]))), p_string(<<"#">>)]))(I,D) end, fun(Node, _Idx) -> + used_combinator(p_regexp), + ["p_regexp(<<\"", + % Escape \ and " as they are used in erlang string. Other sumbol stay as is. + % \ -> \\ + % " -> \" + re:replace(proplists:get_value(string, Node), "\"|\\\\", "\\\\&", [{return, binary}, global]), + "\">>)"] + end). + +-spec 'quoted_string'(input(), index()) -> parse_result(). +'quoted_string'(Input, Index) -> + p(Input, Index, 'quoted_string', fun(I,D) -> (p_choose([fun 'single_quoted_string'/2, fun 'double_quoted_string'/2]))(I,D) end, fun(Node, _Idx) -> + used_combinator(p_string), + lists:flatten(["p_string(<<\"", + escape_string(unicode:characters_to_list(proplists:get_value(string, Node))), + "\">>)"]) + end). + +-spec 'double_quoted_string'(input(), index()) -> parse_result(). +'double_quoted_string'(Input, Index) -> + p(Input, Index, 'double_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\"">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\"">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\"">>), p_anything()])]))), p_string(<<"\"">>)]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'single_quoted_string'(input(), index()) -> parse_result(). +'single_quoted_string'(Input, Index) -> + p(Input, Index, 'single_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\'">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\'">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\'">>), p_anything()])]))), p_string(<<"\'">>)]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'character_class'(input(), index()) -> parse_result(). +'character_class'(Input, Index) -> + p(Input, Index, 'character_class', fun(I,D) -> (p_seq([p_string(<<"[">>), p_label('characters', p_one_or_more(p_seq([p_not(p_string(<<"]">>)), p_choose([p_seq([p_string(<<"\\\\">>), p_anything()]), p_seq([p_not(p_string(<<"\\\\">>)), p_anything()])])]))), p_string(<<"]">>)]))(I,D) end, fun(Node, _Idx) -> + used_combinator(p_charclass), + ["p_charclass(<<\"[", + escape_string(unicode:characters_to_list(proplists:get_value(characters, Node))), + "]\">>)"] + end). + +-spec 'anything_symbol'(input(), index()) -> parse_result(). +'anything_symbol'(Input, Index) -> + p(Input, Index, 'anything_symbol', fun(I,D) -> (p_string(<<".">>))(I,D) end, fun(_Node, _Idx) -> used_combinator(p_anything), <<"p_anything()">> end). + +-spec 'alpha_char'(input(), index()) -> parse_result(). +'alpha_char'(Input, Index) -> + p(Input, Index, 'alpha_char', fun(I,D) -> (p_charclass(<<"[A-Za-z_]">>))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'alphanumeric_char'(input(), index()) -> parse_result(). +'alphanumeric_char'(Input, Index) -> + p(Input, Index, 'alphanumeric_char', fun(I,D) -> (p_choose([fun 'alpha_char'/2, p_charclass(<<"[0-9]">>)]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'space'(input(), index()) -> parse_result(). +'space'(Input, Index) -> + p(Input, Index, 'space', fun(I,D) -> (p_one_or_more(p_choose([fun 'white'/2, fun 'comment_to_eol'/2])))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'comment_to_eol'(input(), index()) -> parse_result(). +'comment_to_eol'(Input, Index) -> + p(Input, Index, 'comment_to_eol', fun(I,D) -> (p_seq([p_not(p_string(<<"%{">>)), p_string(<<"%">>), p_zero_or_more(p_seq([p_not(p_string(<<"\n">>)), p_anything()]))]))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'white'(input(), index()) -> parse_result(). +'white'(Input, Index) -> + p(Input, Index, 'white', fun(I,D) -> (p_charclass(<<"[\s\t\n\r]">>))(I,D) end, fun(Node, _Idx) ->Node end). + +-spec 'code_block'(input(), index()) -> parse_result(). +'code_block'(Input, Index) -> + p(Input, Index, 'code_block', fun(I,D) -> (p_choose([p_seq([p_string(<<"%{">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\%">>), p_string(<<"$%">>), p_seq([p_not(p_string(<<"%}">>)), p_anything()])]))), p_string(<<"%}">>)]), p_seq([p_string(<<"`">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\`">>), p_string(<<"$`">>), p_seq([p_not(p_string(<<"`">>)), p_anything()])]))), p_string(<<"`">>)]), p_string(<<"~">>)]))(I,D) end, fun(Node, _Idx) -> + case Node of + <<"~">> -> {code, <<"Node">>}; + _ -> {code, proplists:get_value('code', Node)} + end + end). + + + +-file("peg_includes.hrl", 1). +-type index() :: {{line, pos_integer()}, {column, pos_integer()}}. +-type input() :: binary(). +-type parse_failure() :: {fail, term()}. +-type parse_success() :: {term(), input(), index()}. +-type parse_result() :: parse_failure() | parse_success(). +-type parse_fun() :: fun((input(), index()) -> parse_result()). +-type xform_fun() :: fun((input(), index()) -> term()). + +-spec p(input(), index(), atom(), parse_fun(), xform_fun()) -> parse_result(). +p(Inp, StartIndex, Name, ParseFun, TransformFun) -> + case get_memo(StartIndex, Name) of % See if the current reduction is memoized + {ok, Memo} -> %Memo; % If it is, return the stored result + Memo; + _ -> % If not, attempt to parse + Result = case ParseFun(Inp, StartIndex) of + {fail,_} = Failure -> % If it fails, memoize the failure + Failure; + {Match, InpRem, NewIndex} -> % If it passes, transform and memoize the result. + Transformed = TransformFun(Match, StartIndex), + {Transformed, InpRem, NewIndex} + end, + memoize(StartIndex, Name, Result), + Result + end. + +-spec setup_memo() -> ets:tid(). +setup_memo() -> + put({parse_memo_table, ?MODULE}, ets:new(?MODULE, [set])). + +-spec release_memo() -> true. +release_memo() -> + ets:delete(memo_table_name()). + +-spec memoize(index(), atom(), parse_result()) -> true. +memoize(Index, Name, Result) -> + Memo = case ets:lookup(memo_table_name(), Index) of + [] -> []; + [{Index, Plist}] -> Plist + end, + ets:insert(memo_table_name(), {Index, [{Name, Result}|Memo]}). + +-spec get_memo(index(), atom()) -> {ok, term()} | {error, not_found}. +get_memo(Index, Name) -> + case ets:lookup(memo_table_name(), Index) of + [] -> {error, not_found}; + [{Index, Plist}] -> + case proplists:lookup(Name, Plist) of + {Name, Result} -> {ok, Result}; + _ -> {error, not_found} + end + end. + +-spec memo_table_name() -> ets:tid(). +memo_table_name() -> + get({parse_memo_table, ?MODULE}). + +-ifdef(p_eof). +-spec p_eof() -> parse_fun(). +p_eof() -> + fun(<<>>, Index) -> {eof, [], Index}; + (_, Index) -> {fail, {expected, eof, Index}} end. +-endif. + +-ifdef(p_optional). +-spec p_optional(parse_fun()) -> parse_fun(). +p_optional(P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} -> {[], Input, Index}; + {_, _, _} = Success -> Success + end + end. +-endif. + +-ifdef(p_not). +-spec p_not(parse_fun()) -> parse_fun(). +p_not(P) -> + fun(Input, Index)-> + case P(Input,Index) of + {fail,_} -> + {[], Input, Index}; + {Result, _, _} -> {fail, {expected, {no_match, Result},Index}} + end + end. +-endif. + +-ifdef(p_assert). +-spec p_assert(parse_fun()) -> parse_fun(). +p_assert(P) -> + fun(Input,Index) -> + case P(Input,Index) of + {fail,_} = Failure-> Failure; + _ -> {[], Input, Index} + end + end. +-endif. + +-ifdef(p_seq). +-spec p_seq([parse_fun()]) -> parse_fun(). +p_seq(P) -> + fun(Input, Index) -> + p_all(P, Input, Index, []) + end. + +-spec p_all([parse_fun()], input(), index(), [term()]) -> parse_result(). +p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index}; +p_all([P|Parsers], Inp, Index, Accum) -> + case P(Inp, Index) of + {fail, _} = Failure -> Failure; + {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum]) + end. +-endif. + +-ifdef(p_choose). +-spec p_choose([parse_fun()]) -> parse_fun(). +p_choose(Parsers) -> + fun(Input, Index) -> + p_attempt(Parsers, Input, Index, none) + end. + +-spec p_attempt([parse_fun()], input(), index(), none | parse_failure()) -> parse_result(). +p_attempt([], _Input, _Index, Failure) -> Failure; +p_attempt([P|Parsers], Input, Index, FirstFailure)-> + case P(Input, Index) of + {fail, _} = Failure -> + case FirstFailure of + none -> p_attempt(Parsers, Input, Index, Failure); + _ -> p_attempt(Parsers, Input, Index, FirstFailure) + end; + Result -> Result + end. +-endif. + +-ifdef(p_zero_or_more). +-spec p_zero_or_more(parse_fun()) -> parse_fun(). +p_zero_or_more(P) -> + fun(Input, Index) -> + p_scan(P, Input, Index, []) + end. +-endif. + +-ifdef(p_one_or_more). +-spec p_one_or_more(parse_fun()) -> parse_fun(). +p_one_or_more(P) -> + fun(Input, Index)-> + Result = p_scan(P, Input, Index, []), + case Result of + {[_|_], _, _} -> + Result; + _ -> + {fail, {expected, Failure, _}} = P(Input,Index), + {fail, {expected, {at_least_one, Failure}, Index}} + end + end. +-endif. + +-ifdef(p_label). +-spec p_label(atom(), parse_fun()) -> parse_fun(). +p_label(Tag, P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} = Failure -> + Failure; + {Result, InpRem, NewIndex} -> + {{Tag, Result}, InpRem, NewIndex} + end + end. +-endif. + +-ifdef(p_scan). +-spec p_scan(parse_fun(), input(), index(), [term()]) -> {[term()], input(), index()}. +p_scan(_, <<>>, Index, Accum) -> {lists:reverse(Accum), <<>>, Index}; +p_scan(P, Inp, Index, Accum) -> + case P(Inp, Index) of + {fail,_} -> {lists:reverse(Accum), Inp, Index}; + {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum]) + end. +-endif. + +-ifdef(p_string). +-spec p_string(binary()) -> parse_fun(). +p_string(S) -> + Length = erlang:byte_size(S), + fun(Input, Index) -> + try + <> = Input, + {S, Rest, p_advance_index(S, Index)} + catch + error:{badmatch,_} -> {fail, {expected, {string, S}, Index}} + end + end. +-endif. + +-ifdef(p_anything). +-spec p_anything() -> parse_fun(). +p_anything() -> + fun(<<>>, Index) -> {fail, {expected, any_character, Index}}; + (Input, Index) when is_binary(Input) -> + <> = Input, + {<>, Rest, p_advance_index(<>, Index)} + end. +-endif. + +-ifdef(p_charclass). +-spec p_charclass(string() | binary()) -> parse_fun(). +p_charclass(Class) -> + {ok, RE} = re:compile(Class, [unicode, dotall]), + fun(Inp, Index) -> + case re:run(Inp, RE, [anchored]) of + {match, [{0, Length}|_]} -> + {Head, Tail} = erlang:split_binary(Inp, Length), + {Head, Tail, p_advance_index(Head, Index)}; + _ -> {fail, {expected, {character_class, binary_to_list(Class)}, Index}} + end + end. +-endif. + +-ifdef(p_regexp). +-spec p_regexp(binary()) -> parse_fun(). +p_regexp(Regexp) -> + {ok, RE} = re:compile(Regexp, [unicode, dotall, anchored]), + fun(Inp, Index) -> + case re:run(Inp, RE) of + {match, [{0, Length}|_]} -> + {Head, Tail} = erlang:split_binary(Inp, Length), + {Head, Tail, p_advance_index(Head, Index)}; + _ -> {fail, {expected, {regexp, binary_to_list(Regexp)}, Index}} + end + end. +-endif. + +-ifdef(line). +-spec line(index() | term()) -> pos_integer() | undefined. +line({{line,L},_}) -> L; +line(_) -> undefined. +-endif. + +-ifdef(column). +-spec column(index() | term()) -> pos_integer() | undefined. +column({_,{column,C}}) -> C; +column(_) -> undefined. +-endif. + +-spec p_advance_index(input() | unicode:charlist() | pos_integer(), index()) -> index(). +p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings + lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput)); +p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters + {{line, Line}, {column, Col}} = Index, + case MatchedInput of + $\n -> {{line, Line+1}, {column, 1}}; + _ -> {{line, Line}, {column, Col+1}} + end. diff --git a/inttest/neotoma1/neotoma_src_rt.erl b/inttest/neotoma1/neotoma_src_rt.erl new file mode 100644 index 00000000..6f7c6fff --- /dev/null +++ b/inttest/neotoma1/neotoma_src_rt.erl @@ -0,0 +1,81 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2015 Luis Rascao (luis.rascao@gmail.com) +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- +-module(neotoma_src_rt). + +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("deps/retest/include/retest.hrl"). + +-define(GENERATED_MODULES, + [csv_peg]). + +files() -> + [{copy, "../../rebar", "rebar"}, + {copy, "rebar.config", "rebar.config"}, + {copy, "mock", "deps"}, + {copy, "src", "src"}]. + +run(Dir) -> + retest_log:log(debug, "Running in Dir: ~s~n", [Dir]), + ?assertMatch({ok, _}, retest:sh("./rebar compile", + [{async, false}])), + + ok = check_files_generated(), + + retest_log:log(debug, "Verify cleanup~n", []), + ?assertMatch({ok, _}, retest_sh:run("./rebar clean", + [])), + ok = check_files_deleted(), + ok. + +check_files_generated() -> + check(fun filelib:is_regular/1, + generated_erl_files()). + +check_files_deleted() -> + check(fun file_does_not_exist/1, + generated_erl_files()). + +generated_erl_files() -> + add_dir("src", add_ext(?GENERATED_MODULES, ".erl")). + +add_ext(Modules, Ext) -> + [lists:concat([Module, Ext]) || Module <- Modules]. + +add_dir(Dir, Files) -> + [filename:join(Dir, File) || File <- Files]. + +file_does_not_exist(F) -> + not filelib:is_regular(F). + +check(Check, Files) -> + lists:foreach( + fun(F) -> + ?assertMatch({true, _}, {Check(F), F}) + end, + Files). diff --git a/inttest/neotoma1/rebar.config b/inttest/neotoma1/rebar.config new file mode 100644 index 00000000..8757da83 --- /dev/null +++ b/inttest/neotoma1/rebar.config @@ -0,0 +1,15 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 ft=erlang et + +{deps, + [ + %% The dependency below to neotoma is needed for "rebar compile" to + %% work, thus for the inttest to work, but the neotoma that is actually + %% used in inttest is brought in from the inttest/neotoma1/mock + %% subdirectory. + {neotoma, ".*"} + ]}. + +{neotoma_opts, [ + {module_ext, "_peg"} +]}. diff --git a/inttest/neotoma1/src/csv.peg b/inttest/neotoma1/src/csv.peg new file mode 100644 index 00000000..000c00f5 --- /dev/null +++ b/inttest/neotoma1/src/csv.peg @@ -0,0 +1,31 @@ +rows <- head:row tail:(crlf row)* / '' +` +case Node of + [] -> []; + [""] -> []; + _ -> + Head = proplists:get_value(head, Node), + Tail = [R || [_,R] <- proplists:get_value(tail, Node)], + [Head|Tail] +end +`; + +row <- head:field tail:(field_sep field)* / '' +` +case Node of + [] -> []; + [""] -> []; + _ -> + Head = proplists:get_value(head, Node), + Tail = [F || [_,F] <- proplists:get_value(tail, Node)], + [Head|Tail] +end +`; +field <- quoted_field / (!field_sep !crlf .)* `iolist_to_binary(Node)`; +quoted_field <- '"' string:('""' / (!'"' .))* '"' +` + String = proplists:get_value(string, Node), + re:replace(String, "[\"]{2}", "\"",[global, {return, binary}]) +`; +field_sep <- ',' ~; +crlf <- [\r]? [\n] ~; diff --git a/inttest/neotoma1/src/neotoma1.app.src b/inttest/neotoma1/src/neotoma1.app.src new file mode 100644 index 00000000..96a50e59 --- /dev/null +++ b/inttest/neotoma1/src/neotoma1.app.src @@ -0,0 +1,6 @@ +{application, neotoma1, + [{vsn, "1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}]}. + From 52b31d8da539805fb6db7a8829e1b60f46865427 Mon Sep 17 00:00:00 2001 From: "John R. Daily" Date: Mon, 3 Aug 2015 10:40:39 -0400 Subject: [PATCH 37/89] Automatically clean neotoma-generated erl files Add myself to THANKS --- THANKS | 1 + src/rebar_neotoma_compiler.erl | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 0d4b5ac1..b4ae84ae 100644 --- a/THANKS +++ b/THANKS @@ -146,3 +146,4 @@ Tony Rogvall Andrey Teplyashin Duncan McGreggor Sebastien Serre +John Daily diff --git a/src/rebar_neotoma_compiler.erl b/src/rebar_neotoma_compiler.erl index 5549dc49..f93d492e 100644 --- a/src/rebar_neotoma_compiler.erl +++ b/src/rebar_neotoma_compiler.erl @@ -40,7 +40,7 @@ %% source_ext: extension of peg source files -module(rebar_neotoma_compiler). --export([compile/2]). +-export([compile/2, clean/2]). %% for internal use only -export([info/2]). @@ -59,6 +59,20 @@ compile(Config, _AppFile) -> option(module_ext, NeoOpts) ++ ".erl", fun compile_neo/3, [{check_last_mod, true}]). +-spec clean(rebar_config:config(), file:filename()) -> 'ok'. +clean(Config, _AppFile) -> + NeoOpts = neotoma_opts(Config), + GeneratedFiles = neotoma_generated_files( + option(out_dir, NeoOpts), + option(module_ext, NeoOpts), + neotoma_source_files( + option(doc_root, NeoOpts), + option(source_ext, NeoOpts) + ) + ), + ok = rebar_file_utils:delete_each(GeneratedFiles), + ok. + %% ============================================================================ %% Internal functions %% ============================================================================ @@ -161,3 +175,20 @@ referenced_pegs1(Step, Config, Seen) -> _ -> referenced_pegs1(sets:to_list(New), Config, sets:union(New, Seen)) end. + +neotoma_source_files(SrcDir, Ext) -> + lists:map( + fun filename:basename/1, + filelib:wildcard(filename:join([SrcDir, "*" ++ Ext])) + ). + + +neotoma_generated_files(OutDir, ModExt, SourceFiles) -> + lists:map( + fun(PegFile) -> + Base = filename:rootname(PegFile), + NewName = Base ++ ModExt ++ ".erl", + filename:join(OutDir, NewName) + end, + SourceFiles + ). From 21f30d3abde6b762e0045687ed0e74682ee2d069 Mon Sep 17 00:00:00 2001 From: Yury Gargay Date: Fri, 12 Feb 2016 18:29:25 +0600 Subject: [PATCH 38/89] Ensure ebin created for dia compiler also add myself to THANKS --- THANKS | 1 + src/rebar_dia_compiler.erl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/THANKS b/THANKS index b4ae84ae..630502a2 100644 --- a/THANKS +++ b/THANKS @@ -147,3 +147,4 @@ Andrey Teplyashin Duncan McGreggor Sebastien Serre John Daily +Yury Gargay diff --git a/src/rebar_dia_compiler.erl b/src/rebar_dia_compiler.erl index 56d51896..a45cd551 100644 --- a/src/rebar_dia_compiler.erl +++ b/src/rebar_dia_compiler.erl @@ -88,6 +88,8 @@ info_help(Description) -> compile_dia(Source, Target, Config) -> ok = filelib:ensure_dir(Target), ok = filelib:ensure_dir(filename:join("include", "dummy.hrl")), + ok = filelib:ensure_dir(filename:join("ebin", "dummy.beam")), + true = code:add_path(filename:absname("ebin")), Opts = [{outdir, "src"}] ++ rebar_config:get(Config, dia_opts, []), case diameter_dict_util:parse({path, Source}, []) of {ok, Spec} -> From 4bc4054abaff9c3445efd5f87f09eae1aefd43b1 Mon Sep 17 00:00:00 2001 From: Yury Gargay Date: Sat, 13 Feb 2016 15:37:16 +0600 Subject: [PATCH 39/89] Add tests for dia compiler Test will be runed only on OTP >= 15 --- test/rebar_dia_tests.erl | 116 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 test/rebar_dia_tests.erl diff --git a/test/rebar_dia_tests.erl b/test/rebar_dia_tests.erl new file mode 100644 index 00000000..556a552d --- /dev/null +++ b/test/rebar_dia_tests.erl @@ -0,0 +1,116 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar_dia_tests). + +-compile([export_all]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(REBAR_SCRIPT, "../rebar"). + +-define(TMP_DIR, "dia_project/"). + +dia_test_() -> + case supported_otp_version() of + true -> + {"Test the dia compiler", + setup, fun() -> setup_project(), rebar("compile") end, + fun teardown/1, + fun(RebarOut) -> + [{"Check ebin is created", + ?_assert(filelib:is_dir("ebin") =:= true)}, + + {"Check include is created", + ?_assert(filelib:is_dir("include") =:= true)}, + + {"Check dia/a.dia is compiled", + ?_assert(string:str(RebarOut, "Compiled dia/a.dia") /= 0)}, + + {"Check dia/b.dia is compiled", + ?_assert(string:str(RebarOut, "Compiled dia/b.dia") /= 0)} + ] + end}; + false -> {setup, fun() -> ok end, []} + end. + + +%% ==================================================================== +%% Setup and Teardown +%% ==================================================================== + +-define(myapp_dia_a, + ["@id 0\n", + "@name a\n", + "@vendor 1 ABC\n", + "@avp_types\n", + " IMSI 1 UTF8String V" + ]). + +-define(myapp_dia_b, + ["@id 2\n", + "@name b\n", + "@vendor 2 ABC\n", + "@inherits a\n", + "@avp_types\n", + " IMEIV 900 OctetString MV" + ]). + +-define(myapp_rebarconfig, + ["{dia_first_files, [\"dia/a.dia\"]}.\n", + "{dia_opts, []}.\n" + ]). + +supported_otp_version() -> + Min = rebar_require_vsn:version_tuple(keep_going, "15", "configured"), + Otp = rebar_require_vsn:version_tuple(keep_going, + erlang:system_info(otp_release), + "OTP Release"), + Otp >= Min. + + +setup_environment() -> + ok = file:make_dir(?TMP_DIR), + prepare_rebar_script(), + ok = file:set_cwd(?TMP_DIR). + +prepare_project() -> + setup_environment(), + rebar("create-app appid=myapp"). + + +setup_project() -> + prepare_project(), + ok = file:make_dir("dia"), + ok = file:write_file("dia/a.dia", ?myapp_dia_a), + ok = file:write_file("dia/b.dia", ?myapp_dia_b), + ok = file:write_file("rebar.config", ?myapp_rebarconfig). + + +teardown(_) -> + ok = file:set_cwd(".."), + ok = remove_tmp_dir(). + +remove_tmp_dir() -> + ok = rebar_file_utils:rm_rf(?TMP_DIR). + +%% ==================================================================== +%% Helper Functions +%% ==================================================================== + +prepare_rebar_script() -> + Rebar = ?TMP_DIR ++ "rebar", + {ok, _} = file:copy(?REBAR_SCRIPT, Rebar), + case os:type() of + {unix, _} -> + [] = os:cmd("chmod u+x " ++ Rebar); + {win32, _} -> + {ok, _} = file:copy(?REBAR_SCRIPT ++ ".cmd", + ?TMP_DIR ++ "rebar.cmd") + end. + +rebar() -> + rebar([]). + +rebar(Args) when is_list(Args) -> + Out = os:cmd(filename:nativename("./rebar") ++ " " ++ Args), + Out. From bffc73d0eef22b4d777049b2ed1a339e3402f61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Sun, 14 Feb 2016 23:40:35 +0000 Subject: [PATCH 40/89] Update regexp to account for newer OTP versions Add support for patch numbers, also expose version_tuple/1 for easier internal access. Normalize version_tuple/1 return value, now always {Major, Minor, Patch}. Update regression tests accordingly. --- src/rebar_require_vsn.erl | 20 +++++++++++------ test/rebar_require_vsn_tests.erl | 37 ++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/rebar_require_vsn.erl b/src/rebar_require_vsn.erl index 5eecf747..684578a7 100644 --- a/src/rebar_require_vsn.erl +++ b/src/rebar_require_vsn.erl @@ -31,7 +31,8 @@ -include("rebar.hrl"). -export([compile/2, - eunit/2]). + eunit/2, + version_tuple/1]). %% for internal use only -export([info/2, @@ -103,12 +104,12 @@ check_versions(Config) -> case rebar_config:get(Config, require_min_otp_vsn, undefined) of undefined -> ?DEBUG("Min OTP version unconfigured~n", []); MinOtpVsn -> - {MinMaj, MinMin} = version_tuple(ShouldAbort, MinOtpVsn, + {MinMaj, MinMin, MinPatch} = version_tuple(ShouldAbort, MinOtpVsn, "configured"), - {OtpMaj, OtpMin} = version_tuple(ShouldAbort, + {OtpMaj, OtpMin, OtpPatch} = version_tuple(ShouldAbort, erlang:system_info(otp_release), "OTP Release"), - case {OtpMaj, OtpMin} >= {MinMaj, MinMin} of + case {OtpMaj, OtpMin, OtpPatch} >= {MinMaj, MinMin, MinPatch} of true -> ?DEBUG("~s satisfies the requirement for vsn ~s~n", [erlang:system_info(otp_release), @@ -122,12 +123,17 @@ check_versions(Config) -> end end. +version_tuple(OtpRelease) -> + version_tuple(keep_going, OtpRelease, ""). + version_tuple(ShouldAbort, OtpRelease, Type) -> - case re:run(OtpRelease, "R?(\\d+)B?-?(\\d+)?", [{capture, all, list}]) of + case re:run(OtpRelease, "R?(\\d+)B?\.?(\\d+)?\.?-?(\\d+)?", [{capture, all, list}]) of + {match, [_Full, Maj, Min, Patch]} -> + {list_to_integer(Maj), list_to_integer(Min), list_to_integer(Patch)}; {match, [_Full, Maj, Min]} -> - {list_to_integer(Maj), list_to_integer(Min)}; + {list_to_integer(Maj), list_to_integer(Min), 0}; {match, [_Full, Maj]} -> - {list_to_integer(Maj), 0}; + {list_to_integer(Maj), 0, 0}; nomatch -> maybe_abort(ShouldAbort, "Cannot parse ~s version string: ~s~n", diff --git a/test/rebar_require_vsn_tests.erl b/test/rebar_require_vsn_tests.erl index a4875866..a934e8e7 100644 --- a/test/rebar_require_vsn_tests.erl +++ b/test/rebar_require_vsn_tests.erl @@ -8,17 +8,32 @@ version_tuple_test_() -> [%% typical cases - ?_assert(check("R15B", "eunit") =:= {15, 0}), - ?_assert(check("R15B01", "eunit") =:= {15, 1}), - ?_assert(check("R15B02", "eunit") =:= {15, 2}), - ?_assert(check("R15B03-1", "eunit") =:= {15, 3}), - ?_assert(check("R15B03", "eunit") =:= {15, 3}), - ?_assert(check("R16B", "eunit") =:= {16, 0}), - ?_assert(check("R16B01", "eunit") =:= {16, 1}), - ?_assert(check("R16B02", "eunit") =:= {16, 2}), - ?_assert(check("R16B03", "eunit") =:= {16, 3}), - ?_assert(check("R16B03-1", "eunit") =:= {16, 3}), - ?_assert(check("17", "eunit") =:= {17, 0}), + ?_assert(check("R14A", "eunit") =:= {14, 0, 0}), + ?_assert(check("R14B", "eunit") =:= {14, 0, 0}), + ?_assert(check("R14B01", "eunit") =:= {14, 1, 0}), + ?_assert(check("R14B02", "eunit") =:= {14, 2, 0}), + ?_assert(check("R14B03", "eunit") =:= {14, 3, 0}), + ?_assert(check("R14B04", "eunit") =:= {14, 4, 0}), + ?_assert(check("R15B", "eunit") =:= {15, 0, 0}), + ?_assert(check("R15B01", "eunit") =:= {15, 1, 0}), + ?_assert(check("R15B02", "eunit") =:= {15, 2, 0}), + ?_assert(check("R15B03-1", "eunit") =:= {15, 3, 1}), + ?_assert(check("R15B03", "eunit") =:= {15, 3, 0}), + ?_assert(check("R16B", "eunit") =:= {16, 0, 0}), + ?_assert(check("R16B01", "eunit") =:= {16, 1, 0}), + ?_assert(check("R16B02", "eunit") =:= {16, 2, 0}), + ?_assert(check("R16B03", "eunit") =:= {16, 3, 0}), + ?_assert(check("R16B03-1", "eunit") =:= {16, 3, 1}), + ?_assert(check("17", "eunit") =:= {17, 0, 0}), + ?_assert(check("17.0", "eunit") =:= {17, 0, 0}), + ?_assert(check("17.1", "eunit") =:= {17, 1, 0}), + ?_assert(check("17.3", "eunit") =:= {17, 3, 0}), + ?_assert(check("17.4", "eunit") =:= {17, 4, 0}), + ?_assert(check("17.5", "eunit") =:= {17, 5, 0}), + ?_assert(check("18.0", "eunit") =:= {18, 0, 0}), + ?_assert(check("18.1", "eunit") =:= {18, 1, 0}), + ?_assert(check("18.2", "eunit") =:= {18, 2, 0}), + ?_assert(check("18.2.1", "eunit") =:= {18, 2, 1}), %% error cases ?_assertException(throw, rebar_abort, check("", "eunit")), ?_assertException(throw, rebar_abort, check("abc", "eunit")) From b816c69e343c8fd757c59fe8703eeda597f4da0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Fri, 19 Feb 2016 16:52:46 +0000 Subject: [PATCH 41/89] Treat port env vars as expandable only if they self reference Provide additional port test case. Also, update newest OS X versions build flags. --- inttest/port2/c_src/test1.c | 13 +++++++ inttest/port2/c_src/test1.h | 0 inttest/port2/port2_rt.erl | 75 +++++++++++++++++++++++++++++++++++++ inttest/port2/rebar.config | 9 +++++ src/rebar_port_compiler.erl | 10 ++++- 5 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 inttest/port2/c_src/test1.c create mode 100644 inttest/port2/c_src/test1.h create mode 100644 inttest/port2/port2_rt.erl create mode 100644 inttest/port2/rebar.config diff --git a/inttest/port2/c_src/test1.c b/inttest/port2/c_src/test1.c new file mode 100644 index 00000000..f1c8ef8a --- /dev/null +++ b/inttest/port2/c_src/test1.c @@ -0,0 +1,13 @@ +#include "test1.h" + +#ifndef TEST1 +#error TEST1 is not defined! +#endif + +#ifndef TEST2 +#error TEST2 is not defined! +#endif + +#ifndef TEST3 +#error TEST3 is not defined! +#endif diff --git a/inttest/port2/c_src/test1.h b/inttest/port2/c_src/test1.h new file mode 100644 index 00000000..e69de29b diff --git a/inttest/port2/port2_rt.erl b/inttest/port2/port2_rt.erl new file mode 100644 index 00000000..ffe8f231 --- /dev/null +++ b/inttest/port2/port2_rt.erl @@ -0,0 +1,75 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2016 Luis Rascao +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- + +-module(port2_rt). +-export([files/0, + run/1]). + +-include_lib("eunit/include/eunit.hrl"). + +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + +files() -> + [ + {copy, "rebar.config", "rebar.config"}, + {copy, "c_src", "c_src"}, + {create, "ebin/foo.app", app(foo, [])} + ] ++ inttest_utils:rebar_setup(). + +run(_Dir) -> + %% wait a bit for new files to have different timestamps + wait(), + %% test.so is created during first compile + ?assertEqual(0, filelib:last_modified("priv/test.so")), + ?assertMatch({ok, _}, retest_sh:run("./rebar compile", + [{env, [{"ERL_CFLAGS", "-DTEST3=test3"}]}])), + TestSo1 = filelib:last_modified("priv/test" ++ + shared_library_file_extension(os:type())), + ?assert(TestSo1 > 0). + +wait() -> + timer:sleep(1000). + +object_file_extension({win32, nt}) -> ".o"; +object_file_extension(_) -> ".o". + +shared_library_file_extension({win32, nt}) -> ".dll"; +shared_library_file_extension(_) -> ".so". + +%% +%% Generate the contents of a simple .app file +%% +app(Name, Modules) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, Modules}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/inttest/port2/rebar.config b/inttest/port2/rebar.config new file mode 100644 index 00000000..3527ee5c --- /dev/null +++ b/inttest/port2/rebar.config @@ -0,0 +1,9 @@ +{port_specs, [ + {"(darwin|linux|freebsd)", "priv/test.so", + ["c_src/*.c"], [ + {env, [ + {"CFLAGS", "$CFLAGS -DTEST1=test1"}, + {"ERL_CFLAGS", "$ERL_CFLAGS -DTEST2=test2"} + ]} + ]} +]}. diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 906bcb09..7af45ebb 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -495,9 +495,10 @@ expand_command(TmplName, Env, InFiles, OutFile) -> %% %% Given a string, determine if it is expandable -%% +%% A string is defined as expandable if it contains itself +%% (eg. CFLAGS = -m64 $CFLAGS) is_expandable(InStr) -> - case re:run(InStr,"\\\$",[{capture,none}]) of + case re:run(InStr,"\\\$"++InStr,[{capture,none}]) of match -> true; nomatch -> false end. @@ -626,6 +627,11 @@ default_env() -> {"darwin9.*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, {"darwin9.*-64$", "LDFLAGS", "-arch x86_64 $LDFLAGS"}, + %% OS X Lion onwards flags for 64-bit + {"darwin1[0-4].*-64$", "CFLAGS", "-m64 $CFLAGS"}, + {"darwin1[0-4].*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, + {"darwin1[0-4].*-64$", "LDFLAGS", "-arch x86_64 $LDFLAGS"}, + %% OS X Snow Leopard, Lion, and Mountain Lion flags for 32-bit {"darwin1[0-2].*-32", "CFLAGS", "-m32 $CFLAGS"}, {"darwin1[0-2].*-32", "CXXFLAGS", "-m32 $CXXFLAGS"}, From fa571d339630c8f4d10d82a7bf16b8dbf2bc1ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Tue, 19 Jan 2016 10:50:39 +0000 Subject: [PATCH 42/89] Additional Common Test regression tests One test that checks for a failed make, another that detects a failed test run when only one of several suites fails. The ct test fails is only applicable to OTP versions older than R16 (inclusive), this is related to the way older versions handle the test .spec files. --- inttest/ct_make_fails/app.config | 2 ++ inttest/ct_make_fails/ct_make_fails_rt.erl | 31 +++++++++++++++++ inttest/ct_make_fails/rebar.config | 2 ++ inttest/ct_make_fails/test_SUITE.erl | 17 ++++++++++ inttest/ct_test_fails/app.config | 2 ++ inttest/ct_test_fails/ct_test_fails.test.spec | 1 + inttest/ct_test_fails/ct_test_fails_rt.erl | 33 +++++++++++++++++++ inttest/ct_test_fails/rebar.config | 13 ++++++++ inttest/ct_test_fails/test2_SUITE.erl | 9 +++++ inttest/ct_test_fails/test_SUITE.erl | 10 ++++++ 10 files changed, 120 insertions(+) create mode 100644 inttest/ct_make_fails/app.config create mode 100644 inttest/ct_make_fails/ct_make_fails_rt.erl create mode 100644 inttest/ct_make_fails/rebar.config create mode 100644 inttest/ct_make_fails/test_SUITE.erl create mode 100644 inttest/ct_test_fails/app.config create mode 100644 inttest/ct_test_fails/ct_test_fails.test.spec create mode 100644 inttest/ct_test_fails/ct_test_fails_rt.erl create mode 100644 inttest/ct_test_fails/rebar.config create mode 100644 inttest/ct_test_fails/test2_SUITE.erl create mode 100644 inttest/ct_test_fails/test_SUITE.erl diff --git a/inttest/ct_make_fails/app.config b/inttest/ct_make_fails/app.config new file mode 100644 index 00000000..bb718b21 --- /dev/null +++ b/inttest/ct_make_fails/app.config @@ -0,0 +1,2 @@ +%% This file is an application config file, not a CT test config file +[{a1, [{foo, bar}]}]. diff --git a/inttest/ct_make_fails/ct_make_fails_rt.erl b/inttest/ct_make_fails/ct_make_fails_rt.erl new file mode 100644 index 00000000..a7959a7d --- /dev/null +++ b/inttest/ct_make_fails/ct_make_fails_rt.erl @@ -0,0 +1,31 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(ct_make_fails_rt). + +-compile(export_all). + + +files() -> + [{create, "ebin/a1.app", app(a1)}, + {copy, "../../rebar", "rebar"}, + {copy, "rebar.config", "rebar.config"}, + {copy, "app.config", "app.config"}, + {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}]. + +run(_Dir) -> + ok = case catch retest:sh("./rebar compile ct -v") of + {error, {stopped, _}} -> ok; + _ -> expected_to_fail + end. + +%% +%% Generate the contents of a simple .app file +%% +app(Name) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/inttest/ct_make_fails/rebar.config b/inttest/ct_make_fails/rebar.config new file mode 100644 index 00000000..58047baf --- /dev/null +++ b/inttest/ct_make_fails/rebar.config @@ -0,0 +1,2 @@ +{ct_dir, "itest"}. +{ct_extra_params, "-repeat 2 -erl_args -config app"}. diff --git a/inttest/ct_make_fails/test_SUITE.erl b/inttest/ct_make_fails/test_SUITE.erl new file mode 100644 index 00000000..343aa5aa --- /dev/null +++ b/inttest/ct_make_fails/test_SUITE.erl @@ -0,0 +1,17 @@ +-module(test_SUITE). + +-compile(export_all). + +-include_lib("ct.hrl"). + +all() -> + [simple_test, + app_config_file_test]. + +simple_test(Config) -> + io:format("Test: ~p\n" [Config]). + +app_config_file_test(_Config) -> + application:start(a1), + {ok, bar} = application:get_env(a1, foo), + application:stop(a1). diff --git a/inttest/ct_test_fails/app.config b/inttest/ct_test_fails/app.config new file mode 100644 index 00000000..bb718b21 --- /dev/null +++ b/inttest/ct_test_fails/app.config @@ -0,0 +1,2 @@ +%% This file is an application config file, not a CT test config file +[{a1, [{foo, bar}]}]. diff --git a/inttest/ct_test_fails/ct_test_fails.test.spec b/inttest/ct_test_fails/ct_test_fails.test.spec new file mode 100644 index 00000000..7bba687a --- /dev/null +++ b/inttest/ct_test_fails/ct_test_fails.test.spec @@ -0,0 +1 @@ +{suites,"../itest",[test_SUITE, test2_SUITE]}. diff --git a/inttest/ct_test_fails/ct_test_fails_rt.erl b/inttest/ct_test_fails/ct_test_fails_rt.erl new file mode 100644 index 00000000..a556891f --- /dev/null +++ b/inttest/ct_test_fails/ct_test_fails_rt.erl @@ -0,0 +1,33 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(ct_test_fails_rt). + +-compile(export_all). + + +files() -> + [{create, "ebin/a1.app", app(a1)}, + {copy, "../../rebar", "rebar"}, + {copy, "rebar.config", "rebar.config"}, + {copy, "app.config", "app.config"}, + {copy, "ct_test_fails.test.spec", "itest/ct_test_fails.test.spec"}, + {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}, + {copy, "test2_SUITE.erl", "itest/test2_SUITE.erl"}]. + +run(Dir) -> + ok = case catch retest:sh("./rebar compile ct -v 3") of + {error, {stopped, _}} -> ok; + _ -> expected_to_fail + end. + +%% +%% Generate the contents of a simple .app file +%% +app(Name) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/inttest/ct_test_fails/rebar.config b/inttest/ct_test_fails/rebar.config new file mode 100644 index 00000000..84c23ef9 --- /dev/null +++ b/inttest/ct_test_fails/rebar.config @@ -0,0 +1,13 @@ +{ct_dir, ["itest"]}. +{ct_extra_params, "-erl_args -config app"}. + +%% http://erlang.org/doc/apps/common_test/run_test_chapter.html#id77160 +%% Any relative paths specified in the test specification, will be relative to the +%% directory which contains the test specification file, if ct_run -spec TestSpecFile ... +%% or ct:run:test([{spec,TestSpecFile},...]) executes the test. The path will be +%% relative to the top level log directory, if ct:run:testspec(TestSpec) executes the test. +%% however for versions older than R16 what counts is the project root path and not the path +%% of the location of the test spec. This will cause the test to fail since R15/14 can't find the +%% test.spec file. Since we can't change the file we have no choice but to bypass the test +%% completely +{require_min_otp_vsn, "R16B"}. diff --git a/inttest/ct_test_fails/test2_SUITE.erl b/inttest/ct_test_fails/test2_SUITE.erl new file mode 100644 index 00000000..4b9299fe --- /dev/null +++ b/inttest/ct_test_fails/test2_SUITE.erl @@ -0,0 +1,9 @@ +-module(test2_SUITE). + +-compile(export_all). + +all() -> + [simple_test]. + +simple_test(Config) -> + io:format("Test: ~p\n", [Config]). diff --git a/inttest/ct_test_fails/test_SUITE.erl b/inttest/ct_test_fails/test_SUITE.erl new file mode 100644 index 00000000..124fbe8a --- /dev/null +++ b/inttest/ct_test_fails/test_SUITE.erl @@ -0,0 +1,10 @@ +-module(test_SUITE). + +-compile(export_all). + +all() -> + [simple_test]. + +simple_test(Config) -> + io:format("Test: ~p\n", [Config]), + ok = not_ok. From a1d8d3d28631a29e716b44264ddfe7c1303d095b Mon Sep 17 00:00:00 2001 From: Tomas Janousek Date: Fri, 20 Jun 2014 16:58:49 +0200 Subject: [PATCH 43/89] Fix eunit_surefire crash with rebar eunit tests=... This fixes the following crash: > =ERROR REPORT==== 20-Jun-2014::16:35:38 === > Error in process <0.40.0> with exit value: {function_clause,[{eunit_surefire,escape_suitename,[undefined,[]],[{file,"eunit_surefire.erl"},{line,416}]},{eunit_surefire,write_report,2,[{file,"eunit_surefire.erl"},{line,258}]},{lists,foreach,2,[{file,"lists.erl"},{line,1336}]},{... --- src/rebar_eunit.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index f1e2ac38..6026740d 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -461,7 +461,7 @@ make_test_primitives(RawTests) -> %% Generator MakePrimitive(generator, M, F2) end, - [NewFunction|Acc] + [eunit_module_suite(M, NewFunction)|Acc] end, lists:foldl(F, [], RawTests). @@ -473,6 +473,11 @@ pre15b02_eunit_primitive(test, M, F) -> pre15b02_eunit_primitive(generator, M, F) -> {generator, eunit_test:function_wrapper(M, F)}. +% Add a test group for eunit_surefire to be able to deduce the testsuite. +% Calling eunit:test({module, M}) does exactly this as well. +eunit_module_suite(M, X) -> + {"module '" ++ atom_to_list(M) ++ "'", X}. + %% %% == run tests == %% From 6711d688572d8d6bdd63fc99f864314b610ddb5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Fri, 19 Feb 2016 18:49:52 +0000 Subject: [PATCH 44/89] Provide additional test coverage for surefire enabled eunit testing --- inttest/eunit_surefire/eunit_src/bar.erl | 6 +++ inttest/eunit_surefire/eunit_surefire_rt.erl | 40 ++++++++++++++++++++ inttest/eunit_surefire/rebar.config | 7 ++++ inttest/eunit_surefire/src/foo.erl | 10 +++++ 4 files changed, 63 insertions(+) create mode 100644 inttest/eunit_surefire/eunit_src/bar.erl create mode 100644 inttest/eunit_surefire/eunit_surefire_rt.erl create mode 100644 inttest/eunit_surefire/rebar.config create mode 100644 inttest/eunit_surefire/src/foo.erl diff --git a/inttest/eunit_surefire/eunit_src/bar.erl b/inttest/eunit_surefire/eunit_src/bar.erl new file mode 100644 index 00000000..6a80dac3 --- /dev/null +++ b/inttest/eunit_surefire/eunit_src/bar.erl @@ -0,0 +1,6 @@ +-module(bar). + +-include_lib("eunit/include/eunit.hrl"). + +bar_test() -> + ?assert(true). diff --git a/inttest/eunit_surefire/eunit_surefire_rt.erl b/inttest/eunit_surefire/eunit_surefire_rt.erl new file mode 100644 index 00000000..fd26d4bb --- /dev/null +++ b/inttest/eunit_surefire/eunit_surefire_rt.erl @@ -0,0 +1,40 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(eunit_surefire_rt). +-export([files/0, run/1]). + +-include_lib("eunit/include/eunit.hrl"). + +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + +files() -> + [ + {create, "ebin/foo.app", app(foo)}, + {copy, "src", "src"}, + {copy, "eunit_src", "eunit_src"}, + {copy, "rebar.config"} + ] ++ inttest_utils:rebar_setup(). + +run(_Dir) -> + {ok, Output} = retest:sh("./rebar -v eunit tests=bar"), + ?assert(check_output(Output, "bar_test")), + ok. + +check_output(Output, Target) -> + lists:any(fun(Line) -> + string:str(Line, Target) > 0 + end, Output). + +%% +%% Generate the contents of a simple .app file +%% +app(Name) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/inttest/eunit_surefire/rebar.config b/inttest/eunit_surefire/rebar.config new file mode 100644 index 00000000..0172560c --- /dev/null +++ b/inttest/eunit_surefire/rebar.config @@ -0,0 +1,7 @@ +{eunit_compile_opts, [ + {src_dirs, ["eunit_src"]} +]}. + +{eunit_opts, [ + {report, {eunit_surefire, [{dir, "."}]}} +]}. diff --git a/inttest/eunit_surefire/src/foo.erl b/inttest/eunit_surefire/src/foo.erl new file mode 100644 index 00000000..a4c91ba7 --- /dev/null +++ b/inttest/eunit_surefire/src/foo.erl @@ -0,0 +1,10 @@ +-module(foo). + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +foo_test() -> + ?assert(true). + +-endif. From e1f7e6a51f5781c626f8d793337215dd70aa7859 Mon Sep 17 00:00:00 2001 From: Frank Hunleth Date: Sun, 6 Mar 2016 11:04:21 -0500 Subject: [PATCH 45/89] Fix bug when running gcc in cross_sizeof The test program used to determine the word size of a crosscompiler is crafted to return an error, so this changes the logic to expect an error exit. If the crosscompiler actually compiles the test program successfully, that would be remarkable and worthy of investigation. --- THANKS | 1 + inttest/cross1/c_src/test1.c | 1 + inttest/cross1/c_src/test1.h | 0 inttest/cross1/cross1_rt.erl | 66 ++++++++++++++++++++++++++++++++++++ src/rebar_utils.erl | 2 +- 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 inttest/cross1/c_src/test1.c create mode 100644 inttest/cross1/c_src/test1.h create mode 100644 inttest/cross1/cross1_rt.erl diff --git a/THANKS b/THANKS index 630502a2..888cc1dd 100644 --- a/THANKS +++ b/THANKS @@ -148,3 +148,4 @@ Duncan McGreggor Sebastien Serre John Daily Yury Gargay +Frank Hunleth diff --git a/inttest/cross1/c_src/test1.c b/inttest/cross1/c_src/test1.c new file mode 100644 index 00000000..4073ed63 --- /dev/null +++ b/inttest/cross1/c_src/test1.c @@ -0,0 +1 @@ +#include "test1.h" diff --git a/inttest/cross1/c_src/test1.h b/inttest/cross1/c_src/test1.h new file mode 100644 index 00000000..e69de29b diff --git a/inttest/cross1/cross1_rt.erl b/inttest/cross1/cross1_rt.erl new file mode 100644 index 00000000..297f3537 --- /dev/null +++ b/inttest/cross1/cross1_rt.erl @@ -0,0 +1,66 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2016 Luis Rascao +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- + +-module(cross1_rt). +-export([files/0, + run/1]). + +-include_lib("eunit/include/eunit.hrl"). + +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + +files() -> + [ + {copy, "c_src", "c_src"}, + {create, "ebin/foo.app", app(foo, [])} + ] ++ inttest_utils:rebar_setup(). + +run(_Dir) -> + %% we will now determine where gcc is located + %% create a symlink to it in the cwd and change the + %% rebar arch so that rebar is fooled into believing + %% it's doing cross compilation. + {ok, [_, GccLocation]} = retest_sh:run("which gcc", []), + {ok, _} = retest_sh:run(io_lib:format("ln -s ~s unknown-unknown-linux-gnu-gcc", [GccLocation]), + []), + ?assertMatch({ok, _}, retest_sh:run("./rebar compile -vvv", + [{env, [{"PATH", "./:" ++ os:getenv("PATH")}, + {"REBAR_TARGET_ARCH", "unknown-unknown-linux-gnu"}]}])). + +%% +%% Generate the contents of a simple .app file +%% +app(Name, Modules) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, Modules}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 64595a2f..1fb5419e 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -760,7 +760,7 @@ cross_sizeof(Arch, Type) -> >>), Cmd = Compiler ++ " -DTYPE=\""++Type++"\" " ++ TempFile, ShOpts = [{use_stdout, false}, return_on_error], - {ok, Res} = sh(Cmd, ShOpts), + {error, {_,Res}} = sh(Cmd, ShOpts), ok = file:delete(TempFile), case string:tokens(Res, ":") of [_, Ln | _] -> From 4802c0a6cac1aae2a6fc43a57d6cbd5c09fc4050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Fri, 12 Feb 2016 13:15:30 +0000 Subject: [PATCH 46/89] Optionally look for ct .spec files in the ct_dir that was specified Allow to change existing behaviour which is to find all .spec files recursively in the current working directory through a new rebar.config option: {ct_search_specs_from_test_dir, true} This is confusing since the user explicitly stated the location for his spec files and negates the possibility to have different spec'ed suites for different environment. Also fix the node name generation on the ct test run, append a random number that will reduce chances of name collisions on epmd. --- inttest/ct4/ct4_rt.erl | 44 ++++++++++++++++++++++++++++++++++ inttest/ct4/deps/bar.test.spec | 1 + inttest/ct4/foo.test.spec | 1 + inttest/ct4/foo_SUITE.erl | 11 +++++++++ inttest/ct4/rebar.config | 2 ++ rebar.config.sample | 4 ++++ src/rebar_ct.erl | 20 ++++++++++++---- 7 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 inttest/ct4/ct4_rt.erl create mode 100644 inttest/ct4/deps/bar.test.spec create mode 100644 inttest/ct4/foo.test.spec create mode 100644 inttest/ct4/foo_SUITE.erl create mode 100644 inttest/ct4/rebar.config diff --git a/inttest/ct4/ct4_rt.erl b/inttest/ct4/ct4_rt.erl new file mode 100644 index 00000000..bc010e86 --- /dev/null +++ b/inttest/ct4/ct4_rt.erl @@ -0,0 +1,44 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(ct4_rt). + +-compile(export_all). + +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + +files() -> + [ + {create, "ebin/foo.app", app(foo)}, + {copy, "rebar.config", "rebar.config"}, + {copy, "foo.test.spec", "test/foo.test.spec"}, + {copy, "deps/bar.test.spec", "deps/bar.test.spec"}, + {copy, "deps/bar.test.spec", "baz.test.spec"}, + {copy, "foo_SUITE.erl", "test/foo_SUITE.erl"} + ] ++ inttest_utils:rebar_setup(). + +run(_Dir) -> + Ref = retest:sh("./rebar compile ct -vvv", [async]), + {ok, [[CTRunCmd]]} = retest:sh_expect(Ref, "^\"ct_run.*", + [global, {capture, first, binary}]), + {match, _} = re:run(CTRunCmd, "foo.test.spec", [global]), + %% deps/bar.test.spec should be ignored by rebar_ct:collect_glob/3 + nomatch = re:run(CTRunCmd, "bar.test.spec", [global]), + %% baz.test.spec should be also ignored by rebar_ct:collect_glob/3 + %% since we specified in rebar.config that we want to search for + %% ct specs from the test dir + nomatch = re:run(CTRunCmd, "baz.test.spec", [global]), + ok. + +%% +%% Generate the contents of a simple .app file +%% +app(Name) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, []}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/inttest/ct4/deps/bar.test.spec b/inttest/ct4/deps/bar.test.spec new file mode 100644 index 00000000..a16610c6 --- /dev/null +++ b/inttest/ct4/deps/bar.test.spec @@ -0,0 +1 @@ +%% this test spec should be ignored diff --git a/inttest/ct4/foo.test.spec b/inttest/ct4/foo.test.spec new file mode 100644 index 00000000..1850410a --- /dev/null +++ b/inttest/ct4/foo.test.spec @@ -0,0 +1 @@ +{suites, "../", [foo_SUITE]}. diff --git a/inttest/ct4/foo_SUITE.erl b/inttest/ct4/foo_SUITE.erl new file mode 100644 index 00000000..fb4f56a9 --- /dev/null +++ b/inttest/ct4/foo_SUITE.erl @@ -0,0 +1,11 @@ +-module(foo_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +all() -> [simple]. + +simple(Config) -> + io:format("Test: ~p\n", [Config]), + ok. diff --git a/inttest/ct4/rebar.config b/inttest/ct4/rebar.config new file mode 100644 index 00000000..387f8c3d --- /dev/null +++ b/inttest/ct4/rebar.config @@ -0,0 +1,2 @@ +{ct_dir, "test"}. +{ct_search_specs_from_test_dir, true}. diff --git a/rebar.config.sample b/rebar.config.sample index 916e47c5..224c85c1 100644 --- a/rebar.config.sample +++ b/rebar.config.sample @@ -148,6 +148,10 @@ %% Option to use short names (i.e., -sname test) when starting ct {ct_use_short_names, true}. +%% Recursively search for .spec files from the test dir, default +%% is false (ie. the search will be from the current working directory) +{ct_search_specs_from_test_dir, true}. + %% == QuickCheck == %% If qc_mod is unspecified, rebar tries to detect Triq or EQC diff --git a/src/rebar_ct.erl b/src/rebar_ct.erl index cf2059ec..b27f6617 100644 --- a/src/rebar_ct.erl +++ b/src/rebar_ct.erl @@ -66,6 +66,7 @@ info(help, ct) -> " ~p~n" " ~p~n" " ~p~n" + " ~p~n" "Valid command line options:~n" " suites=Suite1,Suite2,...,SuiteN~n" " - run Suite1_SUITE, Suite2_SUITE, ..., SuiteN_SUITE~n" @@ -81,7 +82,8 @@ info(help, ct) -> {ct_dir, "itest"}, {ct_log_dir, "test/logs"}, {ct_extra_params, "-boot start_sasl -s myapp"}, - {ct_use_short_names, true} + {ct_use_short_names, true}, + {ct_search_specs_from_test_dir, false} ]). run_test_if_present(TestDir, LogDir, Config, File) -> @@ -235,7 +237,7 @@ make_cmd(TestDir, RawLogDir, Config) -> CodeDirs = [io_lib:format("\"~s\"", [Dir]) || Dir <- [EbinDir|NonLibCodeDirs]], CodePathString = string:join(CodeDirs, " "), - Cmd = case get_ct_specs(Config, Cwd) of + Cmd = case get_ct_specs(Config, search_ct_specs_from(Cwd, TestDir, Config)) of undefined -> ?FMT("~s" " -pa ~s" @@ -276,10 +278,20 @@ make_cmd(TestDir, RawLogDir, Config) -> RawLog = filename:join(LogDir, "raw.log"), {Cmd, RawLog}. +search_ct_specs_from(Cwd, TestDir, Config) -> + case rebar_config:get_local(Config, ct_search_specs_from_test_dir, false) of + true -> filename:join(Cwd, TestDir); + false -> + Cwd + end. + build_name(Config) -> + %% generate a unique name for our test node, we want + %% to make sure the odds of name clashing are low + Random = integer_to_list(crypto:rand_uniform(0, 10000)), case rebar_config:get_local(Config, ct_use_short_names, false) of - true -> "-sname test"; - false -> " -name test@" ++ net_adm:localhost() + true -> "-sname test" ++ Random; + false -> " -name test" ++ Random ++ "@" ++ net_adm:localhost() end. get_extra_params(Config) -> From 7807105138077d9d68777fd31a75f03401ae8026 Mon Sep 17 00:00:00 2001 From: "Matwey V. Kornilov" Date: Thu, 18 Feb 2016 11:40:36 +0300 Subject: [PATCH 47/89] Introduce REBAR_DEPS_PREFER_LIBS env variable to alter search behaviour When REBAR_DEPS_PREFER_LIBS is set, dependencies with defined sources are allowed to be searched for in system lib directory. Under specific circumstances (i.e. in build environments without networking) it is impossible to fetch deps locally. So, user needs a way to ask rebar to search in system lib directory as well. Signed-off-by: Matwey V. Kornilov --- THANKS | 1 + inttest/tdeps_prefer/a.erl | 4 ++ inttest/tdeps_prefer/a.rebar.config | 4 ++ inttest/tdeps_prefer/b.hrl | 1 + inttest/tdeps_prefer/c.hrl | 1 + inttest/tdeps_prefer/root.rebar.config | 1 + inttest/tdeps_prefer/tdeps_prefer_rt.erl | 68 ++++++++++++++++++++++++ src/rebar.erl | 6 ++- src/rebar_deps.erl | 51 +++++++++++++----- 9 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 inttest/tdeps_prefer/a.erl create mode 100644 inttest/tdeps_prefer/a.rebar.config create mode 100644 inttest/tdeps_prefer/b.hrl create mode 100644 inttest/tdeps_prefer/c.hrl create mode 100644 inttest/tdeps_prefer/root.rebar.config create mode 100644 inttest/tdeps_prefer/tdeps_prefer_rt.erl diff --git a/THANKS b/THANKS index 888cc1dd..ab7935c6 100644 --- a/THANKS +++ b/THANKS @@ -149,3 +149,4 @@ Sebastien Serre John Daily Yury Gargay Frank Hunleth +Matwey Kornilov diff --git a/inttest/tdeps_prefer/a.erl b/inttest/tdeps_prefer/a.erl new file mode 100644 index 00000000..bf413a63 --- /dev/null +++ b/inttest/tdeps_prefer/a.erl @@ -0,0 +1,4 @@ +-module({{module}}). + +-include_lib("b/include/b.hrl"). +-include_lib("c/include/c.hrl"). diff --git a/inttest/tdeps_prefer/a.rebar.config b/inttest/tdeps_prefer/a.rebar.config new file mode 100644 index 00000000..41fe9bc9 --- /dev/null +++ b/inttest/tdeps_prefer/a.rebar.config @@ -0,0 +1,4 @@ +{deps, [ + {c, "1", {git, "../repo/c"}}, + {b, "1", {git, "../repo/b"}} +]}. diff --git a/inttest/tdeps_prefer/b.hrl b/inttest/tdeps_prefer/b.hrl new file mode 100644 index 00000000..9f02fab5 --- /dev/null +++ b/inttest/tdeps_prefer/b.hrl @@ -0,0 +1 @@ +-define(HELLO, hello). diff --git a/inttest/tdeps_prefer/c.hrl b/inttest/tdeps_prefer/c.hrl new file mode 100644 index 00000000..152a99f1 --- /dev/null +++ b/inttest/tdeps_prefer/c.hrl @@ -0,0 +1 @@ +-define(WORLD, world). diff --git a/inttest/tdeps_prefer/root.rebar.config b/inttest/tdeps_prefer/root.rebar.config new file mode 100644 index 00000000..d1c37934 --- /dev/null +++ b/inttest/tdeps_prefer/root.rebar.config @@ -0,0 +1 @@ +{sub_dirs, ["apps/a"]}. diff --git a/inttest/tdeps_prefer/tdeps_prefer_rt.erl b/inttest/tdeps_prefer/tdeps_prefer_rt.erl new file mode 100644 index 00000000..94c9b8fd --- /dev/null +++ b/inttest/tdeps_prefer/tdeps_prefer_rt.erl @@ -0,0 +1,68 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(tdeps_prefer_rt). + +-compile(export_all). + +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + +%% Test REBAR_DEPS_FORCE_LIB +%% A -> [B, C] +%% where B should be found globally and C should be found locally +files() -> + [ + %% A application + {create, "apps/a/ebin/a.app", app(a, [a])}, + {copy, "a.rebar.config", "apps/a/rebar.config"}, + {template, "a.erl", "apps/a/src/a.erl", dict:from_list([{module, a}])}, + + %% B application + {create, "libs/b-1/ebin/b.app", app(b, [])}, + {copy, "b.hrl", "libs/b-1/include/b.hrl"}, + + %% C application + {create, "repo/c/ebin/c.app", app(c, [])}, + {copy, "c.hrl", "repo/c/include/c.hrl"}, + + {copy, "root.rebar.config", "rebar.config"} + ] ++ inttest_utils:rebar_setup(). + +apply_cmds([], _Params) -> + ok; +apply_cmds([Cmd | Rest], Params) -> + io:format("Running: ~s (~p)\n", [Cmd, Params]), + {ok, _} = retest_sh:run(Cmd, Params), + apply_cmds(Rest, Params). + +run(Dir) -> + %% Initialize the c apps as git repos so that dependencies pull + %% properly + GitCmds = ["git init", + "git add -A", + "git config user.email 'tdeps@example.com'", + "git config user.name 'tdeps'", + "git commit -a -m \"Initial Commit\""], + ErlLibs = filename:join(Dir, "libs"), + ok = apply_cmds(GitCmds, [{dir, "repo/c"}]), + Env = [ + {"REBAR_DEPS_PREFER_LIBS", "1"}, + {"ERL_LIBS", ErlLibs} + ], + + {ok, _} = retest_sh:run("./rebar -v get-deps", [{env, Env}]), + {ok, _} = retest_sh:run("./rebar -v compile", [{env, Env}]), + ok. + +%% +%% Generate the contents of a simple .app file +%% +app(Name, Modules) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, Modules}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/src/rebar.erl b/src/rebar.erl index 2fceb195..888f80d9 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -284,7 +284,11 @@ help() -> {"freebsd", compile, "c_src/freebsd_tweaks.sh"}, {eunit, "touch file2.out"}, {compile, "touch postcompile.out"}]} - ]). + ]), + ?CONSOLE( + "Environment variables:~n" + " REBAR_DEPS_PREFER_LIBS to look for dependecies in system libs prior fetching.~n" + "~n", []). %% %% Parse command line arguments using getopt and also filtering out any diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index a6e5b274..995ce977 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -58,16 +58,19 @@ preprocess(Config, _) -> %% used globally since it will be set on the first time through here Config1 = set_shared_deps_dir(Config, get_shared_deps_dir(Config, [])), + %% Check whether user forced deps resolution via system wide libs + Config2 = set_deps_prefer_libs(Config1, get_deps_prefer_libs(Config1, undefined)), + %% Get the list of deps for the current working directory and identify those %% deps that are available/present. - Deps = rebar_config:get_local(Config1, deps, []), - {Config2, {AvailableDeps, MissingDeps}} = find_deps(Config1, find, Deps), + Deps = rebar_config:get_local(Config2, deps, []), + {Config3, {AvailableDeps, MissingDeps}} = find_deps(Config2, find, Deps), ?DEBUG("Available deps: ~p\n", [AvailableDeps]), ?DEBUG("Missing deps : ~p\n", [MissingDeps]), %% Add available deps to code path - Config3 = update_deps_code_path(Config2, AvailableDeps), + Config4 = update_deps_code_path(Config3, AvailableDeps), %% Filtering out 'raw' dependencies so that no commands other than %% deps-related can be executed on their directories. @@ -83,8 +86,8 @@ preprocess(Config, _) -> fun(D, Acc) -> rebar_config:set_skip_dir(Acc, D#dep.dir) end, - Config3, - collect_deps(rebar_utils:get_cwd(), Config3)), + Config4, + collect_deps(rebar_utils:get_cwd(), Config4)), %% Return the empty list, as we don't want anything processed before %% us. {ok, NewConfig, []}; @@ -97,12 +100,12 @@ preprocess(Config, _) -> %% Also, if skip_deps=comma,separated,app,list, then only the given %% dependencies are skipped. NewConfig = - case rebar_config:get_global(Config3, skip_deps, false) of + case rebar_config:get_global(Config4, skip_deps, false) of "true" -> lists:foldl( fun(#dep{dir = Dir}, C) -> rebar_config:set_skip_dir(C, Dir) - end, Config3, AvailableDeps); + end, Config4, AvailableDeps); Apps when is_list(Apps) -> SkipApps = [list_to_atom(App) || App <- string:tokens(Apps, ",")], @@ -112,9 +115,9 @@ preprocess(Config, _) -> true -> rebar_config:set_skip_dir(C, Dir); false -> C end - end, Config3, AvailableDeps); + end, Config4, AvailableDeps); _ -> - Config3 + Config4 end, %% Return all the available dep directories for process @@ -254,7 +257,9 @@ info_help(Description) -> " ~p~n" " ~p~n" "Valid command line options:~n" - " deps_dir=\"deps\" (override default or rebar.config deps_dir)~n", + " deps_dir=\"deps\" (override default or rebar.config deps_dir)~n" + "Environment variables:~n" + " REBAR_DEPS_PREFER_LIBS to look for dependecies in system libs prior fetching.~n", [ Description, {deps_dir, "deps"}, @@ -302,6 +307,18 @@ set_shared_deps_dir(Config, _DepsDir) -> get_shared_deps_dir(Config, Default) -> rebar_config:get_xconf(Config, deps_dir, Default). +set_deps_prefer_libs(Config, undefined) -> + DepsPreferLibs = case os:getenv("REBAR_DEPS_PREFER_LIBS") of + false -> false; + _ -> true + end, + rebar_config:set_xconf(Config, deps_prefer_libs, DepsPreferLibs); +set_deps_prefer_libs(Config, _DepsPreferLibs) -> + Config. + +get_deps_prefer_libs(Config, Default) -> + rebar_config:get_xconf(Config, deps_prefer_libs, Default). + get_deps_dir(Config) -> get_deps_dir(Config, ""). @@ -378,9 +395,15 @@ find_dep(Config, Dep) -> %% e.g. {git, "https://github.com/mochi/mochiweb.git", "HEAD"} %% Deps with a source must be found (or fetched) locally. %% Those without a source may be satisfied from lib dir (get_lib_dir). - find_dep(Config, Dep, Dep#dep.source). - -find_dep(Config, Dep, undefined) -> + DepsPreferLibs = get_deps_prefer_libs(Config, false), + Mode = case {Dep#dep.source, DepsPreferLibs} of + {undefined, _DepsPreferLibs} -> maybe_in_lib; + {_DepSource, true} -> maybe_in_lib; + {_DepSource, false} -> local_only + end, + find_dep(Config, Dep, Mode). + +find_dep(Config, Dep, maybe_in_lib) -> %% 'source' is undefined. If Dep is not satisfied locally, %% go ahead and find it amongst the lib_dir's. case find_dep_in_dir(Config, Dep, get_deps_dir(Config, Dep#dep.app)) of @@ -389,7 +412,7 @@ find_dep(Config, Dep, undefined) -> {Config1, {missing, _}} -> find_dep_in_dir(Config1, Dep, get_lib_dir(Dep#dep.app)) end; -find_dep(Config, Dep, _Source) -> +find_dep(Config, Dep, local_only) -> %% _Source is defined. Regardless of what it is, we must find it %% locally satisfied or fetch it from the original source %% into the project's deps From 16bc8f77254d58913748e3d878b48be290a5fa89 Mon Sep 17 00:00:00 2001 From: "Matwey V. Kornilov" Date: Wed, 2 Mar 2016 16:30:30 +0300 Subject: [PATCH 48/89] Introduce REBAR_VSN_CACHE_FILE env variable to load/save vsn cache When REBAR_VSN_CACHE_FILE is set, then vsn cache is loaded from this file on rebar start and updated when new data. Under specific circumstances (i.e. in build environments), full git tree may not be available, but only its snapshot. We need a way to use preheat vsn cache instead of invocing git command. Signed-off-by: Matwey V. Kornilov --- inttest/vsn_cache/main.erl | 13 +++++ inttest/vsn_cache/vsn_cache_rt.erl | 90 ++++++++++++++++++++++++++++++ src/rebar.erl | 3 +- src/rebar_core.erl | 2 +- src/rebar_utils.erl | 22 +++++++- 5 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 inttest/vsn_cache/main.erl create mode 100644 inttest/vsn_cache/vsn_cache_rt.erl diff --git a/inttest/vsn_cache/main.erl b/inttest/vsn_cache/main.erl new file mode 100644 index 00000000..67b64652 --- /dev/null +++ b/inttest/vsn_cache/main.erl @@ -0,0 +1,13 @@ +-module(main). +-behaviour(application). + +-export([start/0,start/1,start/2,stop/1]). + +start() -> + start(permanent). +start(_Restart) -> + ok. +start(_Type,_Args) -> + ok. +stop(_State) -> + ok. diff --git a/inttest/vsn_cache/vsn_cache_rt.erl b/inttest/vsn_cache/vsn_cache_rt.erl new file mode 100644 index 00000000..b4828880 --- /dev/null +++ b/inttest/vsn_cache/vsn_cache_rt.erl @@ -0,0 +1,90 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(vsn_cache_rt). + +-compile(export_all). + +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + +files() -> + [ + %% Cache save check + {create, "save/src/save.app.src", app(save, [main])}, + {create, "save/vsn_cache_file", ""}, + {copy, "main.erl", "save/src/main.erl"}, + + %% Cache load check + {create, "load/src/load.app.src", app(load, [main])}, + {copy, "main.erl", "load/src/main.erl"} + ] ++ inttest_utils:rebar_setup(). + +apply_cmds([], _Params) -> + ok; +apply_cmds([Cmd | Rest], Params) -> + io:format("Running: ~s (~p)\n", [Cmd, Params]), + {ok, _} = retest_sh:run(Cmd, Params), + apply_cmds(Rest, Params). + +run_save(Dir) -> + GitCmds = ["git init", + "git add -A", + "git config user.email 'vsn_cache@example.com'", + "git config user.name 'vsn_cache'", + "git commit -a -m \"Initial Commit\""], + AppDir = filename:join(Dir, "save"), + EbinDir = filename:join(AppDir, "ebin"), + AppFile = filename:join(EbinDir, "save.app"), + VsnCacheFile = filename:join(AppDir, "vsn_cache_file"), + Env = [{"REBAR_VSN_CACHE_FILE", VsnCacheFile}], + + %% Initialize test git repository + ok = apply_cmds(GitCmds, [{dir, AppDir}]), + %% Compile test project with vsn cache enabled + {ok, _} = retest_sh:run("../rebar -v compile", [{env, Env}, {dir, AppDir}]), + %% Vsn cache file has an entry + {ok, [{{git, AppDir}, Hash}]} = file:consult(VsnCacheFile), + %% This vsn entry must coincide with entry from ebin/save.app + {ok, [{application, save, PropList}]} = file:consult(AppFile), + Hash = proplists:get_value(vsn, PropList), + ok. + +run_load(Dir) -> + AppDir = filename:join(Dir, "load"), + EbinDir = filename:join(AppDir, "ebin"), + AppFile = filename:join(EbinDir, "load.app"), + VsnCacheFile = filename:join(AppDir, "vsn_cache_file"), + Hash = "deadbeef", + CacheEntries = [{{git, AppDir}, Hash}], + Env = [{"REBAR_VSN_CACHE_FILE", VsnCacheFile}], + + %% Initialize dummy vsn cache file + vsn_cache_file(VsnCacheFile, CacheEntries), + %% Compile test project with vsn cache enabled + {ok, _} = retest_sh:run("../rebar -v compile", [{env, Env}, {dir, AppDir}]), + %% This vsn entry in cache file must coincide with entry in ebin/load.app + {ok, [{application, load, PropList}]} = file:consult(AppFile), + Hash = proplists:get_value(vsn, PropList), + ok. + +run(Dir) -> + run_save(Dir), + run_load(Dir), + ok. + +%% +%% Generate the contents of a simple .app file +%% +app(Name, Modules) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, git}, + {modules, Modules}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). + +vsn_cache_file(Name, Entries) -> + file:write_file(Name, + [io_lib:format("~p.~n", [X]) || X <- Entries]). diff --git a/src/rebar.erl b/src/rebar.erl index 888f80d9..5a809d29 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -143,7 +143,7 @@ init_config({Options, _NonOptArgs}) -> %% Keep track of how many operations we do, so we can detect bad commands BaseConfig1 = rebar_config:set_xconf(BaseConfig, operations, 0), %% Initialize vsn cache - rebar_config:set_xconf(BaseConfig1, vsn_cache, dict:new()). + rebar_utils:init_vsn_cache(BaseConfig1). init_config1(BaseConfig) -> %% Determine the location of the rebar executable; important for pulling @@ -288,6 +288,7 @@ help() -> ?CONSOLE( "Environment variables:~n" " REBAR_DEPS_PREFER_LIBS to look for dependecies in system libs prior fetching.~n" + " REBAR_VSN_CACHE_FILE to load vsn cache from and save to specified file.~n" "~n", []). %% diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 0650430a..6cc8d385 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -106,7 +106,7 @@ process_commands([Command | Rest], ParentConfig) -> ParentConfig2), %% Wipe out vsn cache to avoid invalid hits when %% dependencies are updated - rebar_config:set_xconf(ParentConfig3, vsn_cache, dict:new()) + rebar_utils:init_vsn_cache(ParentConfig3) catch throw:rebar_abort -> case rebar_config:get_xconf(ParentConfig1, keep_going, false) of diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 1fb5419e..c3ebfe54 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -67,7 +67,9 @@ processing_base_dir/1, processing_base_dir/2, patch_env/2, - cleanup_code_path/1 + cleanup_code_path/1, + init_vsn_cache/1, + save_vsn_cache/1 ]). %% for internal use only @@ -268,6 +270,23 @@ expand_env_variable(InStr, VarName, RawVarValue) -> re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) end. +init_vsn_cache(Config) -> + init_vsn_cache(Config, os:getenv("REBAR_VSN_CACHE_FILE")). +init_vsn_cache(Config, false) -> + rebar_config:set_xconf(Config, vsn_cache, dict:new()); +init_vsn_cache(Config, CacheFile) -> + {ok, CacheList} = file:consult(CacheFile), + CacheDict = dict:from_list(CacheList), + rebar_config:set_xconf(Config, vsn_cache, CacheDict). + +save_vsn_cache(Config) -> + save_vsn_cache(Config, os:getenv("REBAR_VSN_CACHE_FILE")). +save_vsn_cache(_Config, false) -> + ok; +save_vsn_cache(Config, CacheFile) -> + file:write_file(CacheFile, + [io_lib:format("~p.~n", [X]) || X <- dict:to_list(rebar_config:get_xconf(Config, vsn_cache))]). + vcs_vsn(Config, Vsn, Dir) -> Key = {Vsn, Dir}, Cache = rebar_config:get_xconf(Config, vsn_cache), @@ -276,6 +295,7 @@ vcs_vsn(Config, Vsn, Dir) -> VsnString = vcs_vsn_1(Vsn, Dir), Cache1 = dict:store(Key, VsnString, Cache), Config1 = rebar_config:set_xconf(Config, vsn_cache, Cache1), + save_vsn_cache(Config1), {Config1, VsnString}; {ok, VsnString} -> {Config, VsnString} From 3b345046bde490641866e6098d9145c360b04205 Mon Sep 17 00:00:00 2001 From: Jesse Gumm Date: Sun, 20 Mar 2016 09:18:06 -0500 Subject: [PATCH 49/89] Support rebar3-style deps Example: {git, {appname, "git://something/something", {branch, master}}} ^ | |---- Notice the lack of a version Regex --- inttest/rebar3_deps1/a.erl | 8 ++++ inttest/rebar3_deps1/a.rebar.config | 1 + inttest/rebar3_deps1/b.hrl | 1 + inttest/rebar3_deps1/rebar3_deps1_rt.erl | 59 ++++++++++++++++++++++++ src/rebar_deps.erl | 8 ++++ 5 files changed, 77 insertions(+) create mode 100644 inttest/rebar3_deps1/a.erl create mode 100644 inttest/rebar3_deps1/a.rebar.config create mode 100644 inttest/rebar3_deps1/b.hrl create mode 100644 inttest/rebar3_deps1/rebar3_deps1_rt.erl diff --git a/inttest/rebar3_deps1/a.erl b/inttest/rebar3_deps1/a.erl new file mode 100644 index 00000000..835522ac --- /dev/null +++ b/inttest/rebar3_deps1/a.erl @@ -0,0 +1,8 @@ +-module(a). + +-compile(export_all). + +-include_lib("b/include/b.hrl"). + +hello() -> + io:format("~s\n", [?HELLO]). diff --git a/inttest/rebar3_deps1/a.rebar.config b/inttest/rebar3_deps1/a.rebar.config new file mode 100644 index 00000000..6b20717a --- /dev/null +++ b/inttest/rebar3_deps1/a.rebar.config @@ -0,0 +1 @@ +{deps, [{b, {git, "../repo/b"}}]}. diff --git a/inttest/rebar3_deps1/b.hrl b/inttest/rebar3_deps1/b.hrl new file mode 100644 index 00000000..25dfeda5 --- /dev/null +++ b/inttest/rebar3_deps1/b.hrl @@ -0,0 +1 @@ +-define(HELLO, "Hi From B"). diff --git a/inttest/rebar3_deps1/rebar3_deps1_rt.erl b/inttest/rebar3_deps1/rebar3_deps1_rt.erl new file mode 100644 index 00000000..36f94e3b --- /dev/null +++ b/inttest/rebar3_deps1/rebar3_deps1_rt.erl @@ -0,0 +1,59 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +-module(rebar3_deps1_rt). + +-compile(export_all). + +setup([Target]) -> + retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), + ok. + +%% Test deps with rebar3-type dependencies (that is, dependencies without Regexes) +%% Example: {git, {appname, "git://something/something", {branch, master}}} +files() -> + [ + %% A application + {create, "ebin/a.app", app(a, [a])}, + {copy, "a.rebar.config", "rebar.config"}, + {copy, "a.erl", "src/a.erl"}, + + %% B application + {create, "repo/b/ebin/b.app", app(b, [])}, + {copy, "b.hrl", "repo/b/include/b.hrl"} + + ] ++ inttest_utils:rebar_setup(). + +apply_cmds([], _Params) -> + ok; +apply_cmds([Cmd | Rest], Params) -> + io:format("Running: ~s (~p)\n", [Cmd, Params]), + {ok, _} = retest_sh:run(Cmd, Params), + apply_cmds(Rest, Params). + +run(_Dir) -> + %% Initialize the dep app as git repos so that dependencies pull + %% properly + GitCmds = ["git init", + "git add -A", + "git config user.email 'tdeps@example.com'", + "git config user.name 'tdeps'", + "git commit -a -m \"Initial Commit\""], + apply_cmds(GitCmds, [{dir, "repo/b"}]), + + {ok, _} = retest_sh:run("./rebar get-deps", []), + {ok, _} = retest_sh:run("./rebar compile", []), + + true = filelib:is_regular("ebin/a.beam"), + ok. + +%% +%% Generate the contents of a simple .app file +%% +app(Name, Modules) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, "1"}, + {modules, Modules}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 995ce977..def703b7 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -268,10 +268,16 @@ info_help(Description) -> {rebar, "1.0.*"}, {rebar, ".*", {git, "git://github.com/rebar/rebar.git"}}, + {rebar, + {git, "git://github.com/rebar/rebar.git"}}, {rebar, ".*", {git, "git://github.com/rebar/rebar.git", "Rev"}}, + {rebar, + {git, "git://github.com/rebar/rebar.git", "Rev"}}, {rebar, "1.0.*", {git, "git://github.com/rebar/rebar.git", {branch, "master"}}}, + {rebar, + {git, "git://github.com/rebar/rebar.git", {branch, "master"}}}, {rebar, "1.0.0", {git, "git://github.com/rebar/rebar.git", {tag, "1.0.0"}}}, {rebar, "", @@ -371,6 +377,8 @@ find_deps(Config, read, [], Deps) -> {Config, lists:reverse(Deps)}; find_deps(Config, Mode, [App | Rest], Acc) when is_atom(App) -> find_deps(Config, Mode, [{App, ".*", undefined} | Rest], Acc); +find_deps(Config, Mode, [{App, Source} | Rest], Acc) when is_tuple(Source) -> + find_deps(Config, Mode, [{App, ".*", Source} | Rest], Acc); find_deps(Config, Mode, [{App, VsnRegex} | Rest], Acc) when is_atom(App) -> find_deps(Config, Mode, [{App, VsnRegex, undefined} | Rest], Acc); find_deps(Config, Mode, [{App, VsnRegex, Source} | Rest], Acc) -> From 27d0f6166a1328fdeb1a5974e6b7e91a97e13f56 Mon Sep 17 00:00:00 2001 From: Jesse Gumm Date: Sun, 20 Mar 2016 10:08:05 -0500 Subject: [PATCH 50/89] Add raw to VsnRegex-Free dep --- inttest/rebar3_deps1/a.rebar.config | 5 ++++- inttest/rebar3_deps1/c.txt | 1 + inttest/rebar3_deps1/rebar3_deps1_rt.erl | 6 +++++- src/rebar_deps.erl | 5 +++++ 4 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 inttest/rebar3_deps1/c.txt diff --git a/inttest/rebar3_deps1/a.rebar.config b/inttest/rebar3_deps1/a.rebar.config index 6b20717a..de3abe34 100644 --- a/inttest/rebar3_deps1/a.rebar.config +++ b/inttest/rebar3_deps1/a.rebar.config @@ -1 +1,4 @@ -{deps, [{b, {git, "../repo/b"}}]}. +{deps, [ + {b, {git, "../repo/b"}}, + {c, {git, "../repo/c"}, [raw]} +]}. diff --git a/inttest/rebar3_deps1/c.txt b/inttest/rebar3_deps1/c.txt new file mode 100644 index 00000000..5adcd3d8 --- /dev/null +++ b/inttest/rebar3_deps1/c.txt @@ -0,0 +1 @@ +This is a text file. diff --git a/inttest/rebar3_deps1/rebar3_deps1_rt.erl b/inttest/rebar3_deps1/rebar3_deps1_rt.erl index 36f94e3b..e84b6ba3 100644 --- a/inttest/rebar3_deps1/rebar3_deps1_rt.erl +++ b/inttest/rebar3_deps1/rebar3_deps1_rt.erl @@ -19,7 +19,9 @@ files() -> %% B application {create, "repo/b/ebin/b.app", app(b, [])}, - {copy, "b.hrl", "repo/b/include/b.hrl"} + {copy, "b.hrl", "repo/b/include/b.hrl"}, + + {copy, "c.txt", "repo/c/c.txt"} ] ++ inttest_utils:rebar_setup(). @@ -39,11 +41,13 @@ run(_Dir) -> "git config user.name 'tdeps'", "git commit -a -m \"Initial Commit\""], apply_cmds(GitCmds, [{dir, "repo/b"}]), + apply_cmds(GitCmds, [{dir, "repo/c"}]), {ok, _} = retest_sh:run("./rebar get-deps", []), {ok, _} = retest_sh:run("./rebar compile", []), true = filelib:is_regular("ebin/a.beam"), + true = filelib:is_regular("deps/c/c.txt"), ok. %% diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index def703b7..251bdee4 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -283,6 +283,9 @@ info_help(Description) -> {rebar, "", {git, "git://github.com/rebar/rebar.git", {branch, "master"}}, [raw]}, + {rebar, + {git, "git://github.com/rebar/rebar.git", {branch, "master"}}, + [raw]}, {app_name, ".*", {hg, "https://www.example.org/url"}}, {app_name, ".*", {rsync, "Url"}}, {app_name, ".*", {svn, "https://www.example.org/url"}}, @@ -381,6 +384,8 @@ find_deps(Config, Mode, [{App, Source} | Rest], Acc) when is_tuple(Source) -> find_deps(Config, Mode, [{App, ".*", Source} | Rest], Acc); find_deps(Config, Mode, [{App, VsnRegex} | Rest], Acc) when is_atom(App) -> find_deps(Config, Mode, [{App, VsnRegex, undefined} | Rest], Acc); +find_deps(Config, Mode, [{App, Source, Opts} | Rest], Acc) when is_tuple(Source) -> + find_deps(Config, Mode, [{App, ".*", Source, Opts} | Rest], Acc); find_deps(Config, Mode, [{App, VsnRegex, Source} | Rest], Acc) -> find_deps(Config, Mode, [{App, VsnRegex, Source, []} | Rest], Acc); find_deps(Config, Mode, [{App, VsnRegex, Source, Opts} | Rest], Acc) From a4d835bdea621f6d2ebc463f773fec620ee12581 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 13 Apr 2016 18:26:12 -0500 Subject: [PATCH 51/89] add deprecated message --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a7317baa..54e6ad12 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +**rebar is deprecated and will receive only bug fixes. We encourage you to move to [rebar3](https://github.com/erlang/rebar3).** + +---- + rebar ===== From c43f2e55492aead6d2b5f483771cba0ac9e1bdab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Tue, 7 Jun 2016 23:33:38 +0100 Subject: [PATCH 52/89] Revert 'Treat port env vars as expandable only if they self reference' Regression introduced in b816c69e343c8fd757c59fe8703eeda597f4da0a. --- src/rebar_port_compiler.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 7af45ebb..57253e0b 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -495,10 +495,9 @@ expand_command(TmplName, Env, InFiles, OutFile) -> %% %% Given a string, determine if it is expandable -%% A string is defined as expandable if it contains itself -%% (eg. CFLAGS = -m64 $CFLAGS) +%% is_expandable(InStr) -> - case re:run(InStr,"\\\$"++InStr,[{capture,none}]) of + case re:run(InStr,"\\\$",[{capture,none}]) of match -> true; nomatch -> false end. From 20c9f0b1090b2fb9cd9e2769d5c842a3ceac6a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Sun, 5 Jun 2016 18:08:21 +0100 Subject: [PATCH 53/89] Add necessary missing OSX flags for port driver linking --- src/rebar_port_compiler.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 7af45ebb..0a73e65f 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -625,17 +625,17 @@ default_env() -> %% OS X Leopard flags for 64-bit {"darwin9.*-64$", "CFLAGS", "-m64 $CFLAGS"}, {"darwin9.*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, - {"darwin9.*-64$", "LDFLAGS", "-arch x86_64 $LDFLAGS"}, + {"darwin9.*-64$", "LDFLAGS", "-arch x86_64 -flat_namespace -undefined suppress $LDFLAGS"}, %% OS X Lion onwards flags for 64-bit {"darwin1[0-4].*-64$", "CFLAGS", "-m64 $CFLAGS"}, {"darwin1[0-4].*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, - {"darwin1[0-4].*-64$", "LDFLAGS", "-arch x86_64 $LDFLAGS"}, + {"darwin1[0-4].*-64$", "LDFLAGS", "-arch x86_64 -flat_namespace -undefined suppress $LDFLAGS"}, %% OS X Snow Leopard, Lion, and Mountain Lion flags for 32-bit {"darwin1[0-2].*-32", "CFLAGS", "-m32 $CFLAGS"}, {"darwin1[0-2].*-32", "CXXFLAGS", "-m32 $CXXFLAGS"}, - {"darwin1[0-2].*-32", "LDFLAGS", "-arch i386 $LDFLAGS"}, + {"darwin1[0-2].*-32", "LDFLAGS", "-arch i386 -flat_namespace -undefined suppress $LDFLAGS"}, %% Windows specific flags %% add MS Visual C++ support to rebar on Windows From 46d046e8627ca435180481570393195151130852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Tue, 28 Jun 2016 11:04:50 +0100 Subject: [PATCH 54/89] Prepare release 2.6.2 --- RELEASE-NOTES.md | 29 +++++++++++++++++++++++++++++ ebin/rebar.app | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 329575f4..f6ca12dd 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,32 @@ +# 2.6.2 + +* rebar/532: [Automatically clean neotoma-generated erl files, regression test](https://github.com/rebar/rebar/pull/532) +* rebar/549: [Add Windows continuous integration through AppVeyor](https://github.com/rebar/rebar/pull/549) +* rebar/551: [fprof: document Cachegrind support](https://github.com/rebar/rebar/pull/551) +* rebar/556: [fprof: further document Cachegrind support](https://github.com/rebar/rebar/pull/556) +* rebar/557: [Fix windows eunit tests](https://github.com/rebar/rebar/pull/557) +* rebar/558: [Add qualified name tests docs (see pr #119)](https://github.com/rebar/rebar/pull/558) +* rebar/560: [Lock retest dependency](https://github.com/rebar/rebar/pull/560) +* rebar/562: [Add support for Windows integration testing](https://github.com/rebar/rebar/pull/562) +* rebar/567: [Allow windows release to use etc conf dir](https://github.com/rebar/rebar/pull/567) +* rebar/570: [xref: fix dialyzer warning introduced in 69802f63120](https://github.com/rebar/rebar/pull/570) +* rebar/573: [Provide additional debug logging on relup generation](https://github.com/rebar/rebar/pull/573) +* rebar/578: [Add the latest OTP 18 version to the Travis build](https://github.com/rebar/rebar/pull/578) +* rebar/580: [Additional Common Test regression tests](https://github.com/rebar/rebar/pull/580) +* rebar/581: [Update retest dependency url](https://github.com/rebar/rebar/pull/581) +* rebar/586: [Ensure ebin created for dia compiler](https://github.com/rebar/rebar/pull/586) +* rebar/587: [rebar_require_vsn: Update regexp to account for newer OTP versions](https://github.com/rebar/rebar/pull/587) +* rebar/588: [Look for ct .spec files in the ct_dir that was specified](https://github.com/rebar/rebar/pull/588) +* rebar/589: [Introduce REBAR_DEPS_PREFER_LIBS env variable to alter search behaviour](https://github.com/rebar/rebar/pull/589) +* rebar/590: [Treat port env vars as expandable only if they self reference](https://github.com/rebar/rebar/pull/590) +* rebar/591: [Fix/eunit tests surefire crash](https://github.com/rebar/rebar/pull/591) +* rebar/594: [Introduce REBAR_VSN_CACHE_FILE env variable to load/save vsn cache](https://github.com/rebar/rebar/pull/594) +* rebar/599: [Fix bug when running gcc in cross_sizeof](https://github.com/rebar/rebar/pull/599) +* rebar/602: [Support full-source rebar3-style deps (deps without Version Regexes)](https://github.com/rebar/rebar/pull/602) +* rebar/603: [add deprecated message](https://github.com/rebar/rebar/pull/603) +* rebar/605: [Add necessary OSX flags for port driver linking](https://github.com/rebar/rebar/pull/605) +* rebar/606: [Revert 'Treat port env vars as expandable only if they self reference'](https://github.com/rebar/rebar/pull/606) + # 2.6.1 * rebar/514: [Add license to relnotes tool per reviewer request](https://github.com/rebar/rebar/pull/514) diff --git a/ebin/rebar.app b/ebin/rebar.app index b4de6162..b7be9916 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "2.6.1"}, + {vsn, "2.6.2"}, {modules, [ rebar, rebar_abnfc_compiler, rebar_app_utils, From 5b4a87fa56135eae06cfaf514f341a3c552a4a40 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Tue, 12 Jul 2016 14:32:41 +0200 Subject: [PATCH 55/89] qc: lift experimental notice --- src/rebar_qc.erl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/rebar_qc.erl b/src/rebar_qc.erl index 80b2102e..9fa2465f 100644 --- a/src/rebar_qc.erl +++ b/src/rebar_qc.erl @@ -4,7 +4,7 @@ %% %% rebar: Erlang Build Tools %% -%% Copyright (c) 2011-2014 Tuncer Ayaz +%% Copyright (c) 2011-2016 Tuncer Ayaz %% %% Permission is hereby granted, free of charge, to any person obtaining a copy %% of this software and associated documentation files (the "Software"), to deal @@ -43,16 +43,13 @@ %% =================================================================== qc(Config, _AppFile) -> - ?CONSOLE("NOTICE: Using experimental 'qc' command~n", []), run_qc(Config, qc_opts(Config)). triq(Config, _AppFile) -> - ?CONSOLE("NOTICE: Using experimental 'triq' command~n", []), ok = load_qc_mod(triq), run_qc(Config, qc_opts(Config), triq). eqc(Config, _AppFile) -> - ?CONSOLE("NOTICE: Using experimental 'eqc' command~n", []), ok = load_qc_mod(eqc), run_qc(Config, qc_opts(Config), eqc). From 4ec667a0d2b5e02b44b24f629807a7a2867f4cba Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sun, 17 Jul 2016 22:48:26 +0200 Subject: [PATCH 56/89] port_compiler: consistently format default_env/0 --- src/rebar_port_compiler.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 50093c98..5c612a86 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -573,8 +573,8 @@ default_env() -> Arch = os:getenv("REBAR_TARGET_ARCH"), Vsn = os:getenv("REBAR_TARGET_ARCH_VSN"), [ - {"CC", get_tool(Arch, Vsn,"gcc", "cc")}, - {"CXX", get_tool(Arch, Vsn,"g++", "c++")}, + {"CC", get_tool(Arch, Vsn, "gcc", "cc")}, + {"CXX", get_tool(Arch, Vsn, "g++", "c++")}, {"AR", get_tool(Arch, "ar", "ar")}, {"AS", get_tool(Arch, "as", "as")}, {"CPP", get_tool(Arch, Vsn, "cpp", "cpp")}, From 6eb9e806860129de0184e3d82444043e5055e2b2 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Mon, 25 Jul 2016 14:46:09 +0200 Subject: [PATCH 57/89] Introduce and use compat random number module In order for rebar to work with previous and current OTP releases, we generate the rebar_rnd module on startup. rebar_rnd is generated dynamically to make sure that we use the right random module (either rand if available or else random). It only wraps the common subset of the API, but that's sufficient for rebar's use. --- ebin/rebar.app | 3 +- rebar.config | 4 +- src/rebar.erl | 3 + src/rebar_eunit.erl | 4 +- src/rebar_rand_compat.erl | 178 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 src/rebar_rand_compat.erl diff --git a/ebin/rebar.app b/ebin/rebar.app index b7be9916..4d588bd9 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -46,7 +46,8 @@ rebar_metacmds, rebar_getopt, rebar_mustache, - rmemo ]}, + rmemo, + rebar_rand_compat ]}, {registered, []}, {applications, [ diff --git a/rebar.config b/rebar.config index bcd53685..eda5a2c8 100644 --- a/rebar.config +++ b/rebar.config @@ -30,7 +30,9 @@ - (\"diameter_codegen\":\"from_dict\"/\"4\") - (\"diameter_dict_util\":\"format_error\"/\"1\") - (\"diameter_dict_util\":\"parse\"/\"2\") - - (\"erlang\":\"timestamp\"/\"0\"))", + - (\"erlang\":\"timestamp\"/\"0\") + - (\"rebar_rnd\":\"seed\"/\"1\") + - (\"rebar_rnd\":\"uniform\"/\"0\"))", []}]}. {dialyzer, diff --git a/src/rebar.erl b/src/rebar.erl index 5a809d29..2eb608d0 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -220,6 +220,9 @@ run_aux(BaseConfig, Commands) -> {error, {already_started, _}} -> ok end, + %% Make sure rebar_rnd module is generated, compiled, and loaded + {ok, rebar_rnd} = rebar_rand_compat:init("rebar_rnd"), + %% Convert command strings to atoms CommandAtoms = [list_to_atom(C) || C <- Commands], diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index 6026740d..f4d7b76d 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -293,9 +293,9 @@ randomize_suites(Config, Modules) -> end. randomize_suites1(Modules, Seed) -> - _ = random:seed(35, Seed, 1337), + _ = rebar_rnd:seed({35, Seed, 1337}), ?CONSOLE("Randomizing suite order with seed ~b~n", [Seed]), - [X||{_,X} <- lists:sort([{random:uniform(), M} || M <- Modules])]. + [X||{_,X} <- lists:sort([{rebar_rnd:uniform(), M} || M <- Modules])]. %% %% == get matching tests == diff --git a/src/rebar_rand_compat.erl b/src/rebar_rand_compat.erl new file mode 100644 index 00000000..849ee35e --- /dev/null +++ b/src/rebar_rand_compat.erl @@ -0,0 +1,178 @@ +%%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%%% ex: ts=4 sw=4 et +%%% +%%% Copyright (c) 2016 Tuncer Ayaz +%%% +%%% Permission to use, copy, modify, and/or distribute this software +%%% for any purpose with or without fee is hereby granted, provided +%%% that the above copyright notice and this permission notice appear +%%% in all copies. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +%%% WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +%%% WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +%%% AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +%%% CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +%%% LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +%%% NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +%%% CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +-module(rebar_rand_compat). + +%% API +-export([ init/0 + , init/1 + ]). + +-define(DEFAULT_MODNAME, "rnd"). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% Generate, compile and load rnd module. +%% @end +%%-------------------------------------------------------------------- +-spec init() -> {'ok', module()}. +init() -> + init(?DEFAULT_MODNAME). + +%%-------------------------------------------------------------------- +%% @doc +%% Generate, compile and load ModName module. +%% @end +%%-------------------------------------------------------------------- +-spec init(string()) -> {'ok', module()}. +init(ModName) -> + %% First, select the right module, then generate the appropriate code as a + %% string, and finally compile and load it as ModName. + Src = select(ModName), + {ok, Mod, Bin, []} = compile(Src), + {module, Mod} = code:load_binary(Mod, ModName++".erl", Bin), + {ok, Mod}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +%% Select right rand module and return wrapper module's source as string +-spec select(string()) -> string(). +select(ModName) -> + case code:which(rand) of + non_existing -> + src(ModName, fun funs_random/0); + _ -> + src(ModName, fun funs_rand/0) + end. + +%% Return module's implementation as a string. +-spec src(string(), fun(() -> binary())) -> string(). +src(ModName, GenFuns) -> + lists:flatten( + io_lib:format( + <<" +-module(~s). +-export([ seed/0 + , seed/1 + , uniform/0 + , uniform/1 + , uniform_s/1 + , uniform_s/2 + ]). + +%% Functions +~s +">> +, [ModName, GenFuns()])). + +%% random.beam wrapper +funs_random() -> + <<" +seed() -> random:seed(). +seed(Exp) -> random:seed(Exp). +uniform() -> random:uniform(). +uniform(N) -> random:uniform(N). +uniform_s(St) -> random:uniform_s(St). +uniform_s(N, St) -> random:uniform_s(N, St). +">>. + +%% rand.beam wrapper +funs_rand() -> + <<" +seed() -> rand:seed(exsplus). +seed(Exp) -> rand:seed(exsplus, Exp). +uniform() -> rand:uniform(). +uniform(N) -> rand:uniform(N). +uniform_s(St) -> rand:uniform_s(St). +uniform_s(N, St) -> rand:uniform_s(N, St). +">>. + +compile(String) -> + Forms = convert(String ++ eof, []), + compile:forms(Forms, [return]). + +%% Parse string into forms for compiler. +convert({done, {eof, _EndLocation}, _LeftOverChars}, Acc)-> + %% Finished + lists:reverse(Acc); +convert({done, {error, ErrorInfo, _EndLocation}, _LeftOverChars}, _Acc)-> + ErrorInfo; +convert({done, {ok, Tokens, _EndLocation}, LeftOverChars}, Acc)-> + case erl_parse:parse_form(Tokens) of + {ok, AbsForm} -> + convert(LeftOverChars, [AbsForm|Acc]); + {error, AbsForm} -> + convert(LeftOverChars, AbsForm) + end; +convert({more, Continuation}, Acc)-> + convert(erl_scan:tokens(Continuation, [], 1), Acc); +convert(String, Acc) -> + convert(erl_scan:tokens([], String, 1), Acc). + +%%%=================================================================== +%%% Tests +%%%=================================================================== + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +init_test() -> + DefMod = list_to_atom(?DEFAULT_MODNAME), + ok = unload(DefMod), + ?assertMatch(false, code:is_loaded(DefMod)), + ?assertMatch({ok, DefMod}, init()), + ?assertMatch({file, _}, code:is_loaded(DefMod)), + check_api(DefMod), + CustomMod = foornd, + CustomName = "foornd", + ok = unload(CustomMod), + ?assertMatch(false, code:is_loaded(CustomMod)), + ?assertMatch({ok, CustomMod}, init(CustomName)), + ?assertMatch({file, _}, code:is_loaded(CustomMod)), + check_api(CustomMod). + +unload(Mod) -> + case code:is_loaded(Mod) of + false -> + ok; + {file, _} -> + code:delete(Mod), + code:purge(Mod), + ok + end. + +check_api(Mod) -> + Exports = [ {seed, 0} + , {seed, 1} + , {uniform, 0} + , {uniform, 1} + , {uniform_s, 1} + , {uniform_s, 2} + , {module_info, 0} + , {module_info, 1} + ], + ?assertMatch(Exports, Mod:module_info(exports)). + +-endif. From da6409079e974d8528855a986d5601b52841c4d8 Mon Sep 17 00:00:00 2001 From: Stryder Date: Mon, 15 Aug 2016 15:48:12 -0700 Subject: [PATCH 58/89] Add an additional check for the existence of a 'rebar.beam' file during bootstrap run. Bootstrap fails on first run without any .beam files in the ebin folder. --- bootstrap | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bootstrap b/bootstrap index f02ccd36..d65ccee2 100755 --- a/bootstrap +++ b/bootstrap @@ -24,7 +24,10 @@ main(Args) -> true -> rm("ebin/*.beam"); false -> - rm("ebin/rebar.beam") + case filelib:is_file("ebin/rebar.beam") of + true -> rm("ebin/rebar.beam"); + false -> io:fwrite("No beam files found.~n") + end end, %% Add check for debug flag From 05092202b35430f2b682fd40b7b5f1b3c773bb04 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Thu, 18 Aug 2016 23:52:34 +0200 Subject: [PATCH 59/89] port_compiler: generate clang compilation db In order for newer clang tools to work, they require the presence of a compilation database in the form of compile_commands.json. Therefore, we adapt port_compiler to write such a file. --- inttest/port/port_rt.erl | 1 + src/rebar_port_compiler.erl | 64 +++++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/inttest/port/port_rt.erl b/inttest/port/port_rt.erl index c910e0ed..afe991e1 100644 --- a/inttest/port/port_rt.erl +++ b/inttest/port/port_rt.erl @@ -48,6 +48,7 @@ run(_Dir) -> %% test.so is created during first compile ?assertEqual(0, filelib:last_modified("priv/test.so")), ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), + ?assertMatch(true, filelib:is_regular("compile_commands.json")), TestSo1 = filelib:last_modified("priv/test" ++ shared_library_file_extension(os:type())), ?assert(TestSo1 > 0), diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 5c612a86..10f8f691 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -211,27 +211,65 @@ replace_extension(File, OldExt, NewExt) -> %% compile_sources(Config, Specs, SharedEnv) -> - lists:foldl( - fun(#spec{sources=Sources, type=Type, opts=Opts}, NewBins) -> - Env = proplists:get_value(env, Opts, SharedEnv), - compile_each(Config, Sources, Type, Env, NewBins) - end, [], Specs). - -compile_each(_Config, [], _Type, _Env, NewBins) -> - lists:reverse(NewBins); -compile_each(Config, [Source | Rest], Type, Env, NewBins) -> + {Res, Db} = + lists:foldl( + fun(#spec{sources=Sources, type=Type, opts=Opts}, NewBins) -> + Env = proplists:get_value(env, Opts, SharedEnv), + compile_each(Config, Sources, Type, Env, {NewBins, []}) + end, [], Specs), + %% Rewrite clang compile commands database file only if something + %% was compiled. + case Res of + [] -> + ok; + _ -> + {ok, ClangDbFile} = file:open("compile_commands.json", [write]), + ok = io:fwrite(ClangDbFile, "[~n", []), + lists:foreach(fun(E) -> ok = io:fwrite(ClangDbFile, E, []) end, Db), + ok = io:fwrite(ClangDbFile, "]~n", []), + ok = file:close(ClangDbFile) + end, + Res. + +compile_each(_Config, [], _Type, _Env, {NewBins, CDB}) -> + {lists:reverse(NewBins), lists:reverse(CDB)}; +compile_each(Config, [Source | Rest], Type, Env, {NewBins, CDB}) -> Ext = filename:extension(Source), Bin = replace_extension(Source, Ext, ".o"), + Template = select_compile_template(Type, compiler(Ext)), + Cmd = expand_command(Template, Env, Source, Bin), + %% Omit all variables from cmd, and use that as cmd in + %% CDB, because otherwise clang-* will complain about it. + CDBCmd = string:join( + lists:filter( + fun("$"++_) -> false; + (_) -> true + end, + string:tokens(Cmd, " ")), + " "), + Cwd = rebar_utils:get_cwd(), + %% If there are more source files, make sure we end the CDB entry + %% with a comma. + CDBEntSep = case Rest of + [] -> "~n"; + _ -> ",~n" + end, + %% CDB entry + CDBEnt = ?FMT("{ \"file\" : ~p~n" + ", \"directory\" : ~p~n" + ", \"command\" : ~p~n}" + "~s", + [Source, Cwd, CDBCmd, CDBEntSep] + ), case needs_compile(Source, Bin) of true -> - Template = select_compile_template(Type, compiler(Ext)), - Cmd = expand_command(Template, Env, Source, Bin), ShOpts = [{env, Env}, return_on_error, {use_stdout, false}], exec_compiler(Config, Source, Cmd, ShOpts), - compile_each(Config, Rest, Type, Env, [Bin | NewBins]); + compile_each(Config, Rest, Type, Env, + {[Bin | NewBins], [CDBEnt | CDB]}); false -> ?INFO("Skipping ~s\n", [Source]), - compile_each(Config, Rest, Type, Env, NewBins) + compile_each(Config, Rest, Type, Env, {NewBins, [CDBEnt, CDB]}) end. exec_compiler(Config, Source, Cmd, ShOpts) -> From 4914a2f29ff6d5ced53d5dc7c97b9e82ff7033f8 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sat, 20 Aug 2016 23:15:55 +0200 Subject: [PATCH 60/89] port_compiler: clean up compile_each() Extract clang db entry generation into a separate function. --- src/rebar_port_compiler.erl | 46 ++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 10f8f691..48c276c8 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -238,6 +238,20 @@ compile_each(Config, [Source | Rest], Type, Env, {NewBins, CDB}) -> Bin = replace_extension(Source, Ext, ".o"), Template = select_compile_template(Type, compiler(Ext)), Cmd = expand_command(Template, Env, Source, Bin), + CDBEnt = cdb_entry(Source, Cmd, Rest), + case needs_compile(Source, Bin) of + true -> + ShOpts = [{env, Env}, return_on_error, {use_stdout, false}], + exec_compiler(Config, Source, Cmd, ShOpts), + compile_each(Config, Rest, Type, Env, + {[Bin | NewBins], [CDBEnt | CDB]}); + false -> + ?INFO("Skipping ~s\n", [Source]), + compile_each(Config, Rest, Type, Env, {NewBins, [CDBEnt, CDB]}) + end. + +%% Generate a clang compilation db entry for Src and Cmd +cdb_entry(Src, Cmd, SrcRest) -> %% Omit all variables from cmd, and use that as cmd in %% CDB, because otherwise clang-* will complain about it. CDBCmd = string:join( @@ -246,31 +260,21 @@ compile_each(Config, [Source | Rest], Type, Env, {NewBins, CDB}) -> (_) -> true end, string:tokens(Cmd, " ")), - " "), + " "), + Cwd = rebar_utils:get_cwd(), %% If there are more source files, make sure we end the CDB entry %% with a comma. - CDBEntSep = case Rest of - [] -> "~n"; - _ -> ",~n" - end, + Sep = case SrcRest of + [] -> "~n"; + _ -> ",~n" + end, %% CDB entry - CDBEnt = ?FMT("{ \"file\" : ~p~n" - ", \"directory\" : ~p~n" - ", \"command\" : ~p~n}" - "~s", - [Source, Cwd, CDBCmd, CDBEntSep] - ), - case needs_compile(Source, Bin) of - true -> - ShOpts = [{env, Env}, return_on_error, {use_stdout, false}], - exec_compiler(Config, Source, Cmd, ShOpts), - compile_each(Config, Rest, Type, Env, - {[Bin | NewBins], [CDBEnt | CDB]}); - false -> - ?INFO("Skipping ~s\n", [Source]), - compile_each(Config, Rest, Type, Env, {NewBins, [CDBEnt, CDB]}) - end. + ?FMT("{ \"file\" : ~p~n" + ", \"directory\" : ~p~n" + ", \"command\" : ~p~n" + "}~s", + [Src, Cwd, CDBCmd, Sep]). exec_compiler(Config, Source, Cmd, ShOpts) -> case rebar_utils:sh(Cmd, ShOpts) of From af08edb8707a2b2474a34352c14671b717869c5e Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 26 Aug 2016 06:12:05 +0200 Subject: [PATCH 61/89] port_compiler: auto-select C++ specific link template Even though "$CC foo.o -lstdc++ -o foo.so" works if foo.o was built with $CXX, depending on the platform and toolchain and C++ standard used, it may not. In order to ensure this works reliably, we introduce EXE_LINK_CXX_TEMPLATE and DRV_LINK_CXX_TEMPLATE and auto-select if the port spec being built consists of at least one C++ source. --- src/rebar_port_compiler.erl | 46 ++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 10f8f691..18683d9a 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -36,10 +36,11 @@ -include("rebar.hrl"). -record(spec, {type::'drv' | 'exe', + link_lang::'cc' | 'cxx', target::file:filename(), sources = [] :: [file:filename(), ...], objects = [] :: [file:filename(), ...], - opts = [] ::list() | []}). + opts = [] :: list() | []}). %% =================================================================== %% Public API @@ -65,13 +66,15 @@ compile(Config, AppFile) -> %% Only relink if necessary, given the Target %% and list of new binaries lists:foreach( - fun(#spec{target=Target, objects=Bins, opts=Opts}) -> + fun(#spec{target=Target, objects=Bins, opts=Opts, + link_lang=LinkLang}) -> AllBins = [sets:from_list(Bins), sets:from_list(NewBins)], Intersection = sets:intersection(AllBins), case needs_link(Target, sets:to_list(Intersection)) of true -> - LinkTemplate = select_link_template(Target), + LinkTemplate = select_link_template(LinkLang, + Target), Env = proplists:get_value(env, Opts, SharedEnv), Cmd = expand_command(LinkTemplate, Env, string:join(Bins, " "), @@ -140,12 +143,14 @@ info_help(Description) -> " EXE_CFLAGS - flags that will be used for compiling~n" " EXE_LDFLAGS - flags that will be used for linking~n" " ERL_EI_LIBDIR - ei library directory~n" - " DRV_CXX_TEMPLATE - C++ command template~n" - " DRV_CC_TEMPLATE - C command template~n" - " DRV_LINK_TEMPLATE - Linker command template~n" - " EXE_CXX_TEMPLATE - C++ command template~n" - " EXE_CC_TEMPLATE - C command template~n" - " EXE_LINK_TEMPLATE - Linker command template~n" + " DRV_CXX_TEMPLATE - C++ command template~n" + " DRV_CC_TEMPLATE - C command template~n" + " DRV_LINK_TEMPLATE - C Linker command template~n" + " DRV_LINK_CXX_TEMPLATE - C++ Linker command template~n" + " EXE_CXX_TEMPLATE - C++ command template~n" + " EXE_CC_TEMPLATE - C command template~n" + " EXE_LINK_TEMPLATE - C Linker command template~n" + " EXE_LINK_CXX_TEMPLATE - C++ Linker command template~n" "~n" " Note that if you wish to extend (vs. replace) these variables,~n" " you MUST include a shell-style reference in your definition.~n" @@ -361,6 +366,7 @@ port_spec_from_legacy(Config, AppFile) -> Sources = port_sources(rebar_config:get_list(Config, port_sources, ["c_src/*.c"])), #spec { type = target_type(Target), + link_lang = cc, target = maybe_switch_extension(os:type(), Target), sources = Sources, objects = port_objects(Sources) }. @@ -381,9 +387,17 @@ get_port_spec(Config, OsType, {Arch, Target, Sources}) -> get_port_spec(Config, OsType, {Arch, Target, Sources, []}); get_port_spec(Config, OsType, {_Arch, Target, Sources, Opts}) -> SourceFiles = port_sources(Sources), + LinkLang = + case lists:any( + fun(Src) -> compiler(filename:extension(Src)) == "$CXX" end, + SourceFiles) of + true -> cxx; + false -> cc + end, ObjectFiles = port_objects(SourceFiles), #spec{type=target_type(Target), target=maybe_switch_extension(OsType, Target), + link_lang=LinkLang, sources=SourceFiles, objects=ObjectFiles, opts=port_opts(Config, Opts)}. @@ -586,10 +600,12 @@ select_compile_drv_template("$CXX") -> "DRV_CXX_TEMPLATE". select_compile_exe_template("$CC") -> "EXE_CC_TEMPLATE"; select_compile_exe_template("$CXX") -> "EXE_CXX_TEMPLATE". -select_link_template(Target) -> - case target_type(Target) of - drv -> "DRV_LINK_TEMPLATE"; - exe -> "EXE_LINK_TEMPLATE" +select_link_template(LinkLang, Target) -> + case {LinkLang, target_type(Target)} of + {cc, drv} -> "DRV_LINK_TEMPLATE"; + {cxx, drv} -> "DRV_LINK_CXX_TEMPLATE"; + {cc, exe} -> "EXE_LINK_TEMPLATE"; + {cxx, exe} -> "EXE_LINK_CXX_TEMPLATE" end. target_type(Target) -> target_type1(filename:extension(Target)). @@ -629,12 +645,16 @@ default_env() -> "$CC -c $CFLAGS $DRV_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, {"DRV_LINK_TEMPLATE", "$CC $PORT_IN_FILES $LDFLAGS $DRV_LDFLAGS -o $PORT_OUT_FILE"}, + {"DRV_LINK_CXX_TEMPLATE", + "$CXX $PORT_IN_FILES $LDFLAGS $DRV_LDFLAGS -o $PORT_OUT_FILE"}, {"EXE_CXX_TEMPLATE", "$CXX -c $CXXFLAGS $EXE_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, {"EXE_CC_TEMPLATE", "$CC -c $CFLAGS $EXE_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, {"EXE_LINK_TEMPLATE", "$CC $PORT_IN_FILES $LDFLAGS $EXE_LDFLAGS -o $PORT_OUT_FILE"}, + {"EXE_LINK_CXX_TEMPLATE", + "$CXX $PORT_IN_FILES $LDFLAGS $EXE_LDFLAGS -o $PORT_OUT_FILE"}, {"DRV_CFLAGS" , "-g -Wall -fPIC -MMD $ERL_CFLAGS"}, {"DRV_LDFLAGS", "-shared $ERL_LDFLAGS"}, {"EXE_CFLAGS" , "-g -Wall -fPIC -MMD $ERL_CFLAGS"}, From 57d3c4d172746879ebbd4d8532c7247f7d972264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Sat, 27 Aug 2016 00:44:44 +0100 Subject: [PATCH 62/89] Prepare release 2.6.3 --- RELEASE-NOTES.md | 10 ++++++++++ ebin/rebar.app | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index f6ca12dd..125c2815 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,13 @@ +# 2.6.3 + +* rebar/610: [qc: lift experimental notice](https://github.com/rebar/rebar/pull/610) +* rebar/611: [port_compiler: consistently format default_env/0](https://github.com/rebar/rebar/pull/611) +* rebar/612: [Introduce and use compat random number module](https://github.com/rebar/rebar/pull/612) +* rebar/614: [Add an additional check for the existence of a 'rebar.beam' file](https://github.com/rebar/rebar/pull/614) +* rebar/615: [port_compiler: generate clang compilation db](https://github.com/rebar/rebar/pull/615) +* rebar/616: [port_compiler: clean up compile_each()](https://github.com/rebar/rebar/pull/616) +* rebar/617: [port_compiler: auto-select C++ specific link template](https://github.com/rebar/rebar/pull/617) + # 2.6.2 * rebar/532: [Automatically clean neotoma-generated erl files, regression test](https://github.com/rebar/rebar/pull/532) diff --git a/ebin/rebar.app b/ebin/rebar.app index 4d588bd9..1ad483be 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "2.6.2"}, + {vsn, "2.6.3"}, {modules, [ rebar, rebar_abnfc_compiler, rebar_app_utils, From 4b4d66cc640fa933be398b5dcde1e4aa51a98cc0 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sun, 28 Aug 2016 18:09:44 +0200 Subject: [PATCH 63/89] Fix cdb processing when a file is skipped --- src/rebar_port_compiler.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 1119e4ee..7aa04ec7 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -244,15 +244,16 @@ compile_each(Config, [Source | Rest], Type, Env, {NewBins, CDB}) -> Template = select_compile_template(Type, compiler(Ext)), Cmd = expand_command(Template, Env, Source, Bin), CDBEnt = cdb_entry(Source, Cmd, Rest), + NewCDB = [CDBEnt | CDB], case needs_compile(Source, Bin) of true -> ShOpts = [{env, Env}, return_on_error, {use_stdout, false}], exec_compiler(Config, Source, Cmd, ShOpts), compile_each(Config, Rest, Type, Env, - {[Bin | NewBins], [CDBEnt | CDB]}); + {[Bin | NewBins], NewCDB}); false -> ?INFO("Skipping ~s\n", [Source]), - compile_each(Config, Rest, Type, Env, {NewBins, [CDBEnt, CDB]}) + compile_each(Config, Rest, Type, Env, {NewBins, NewCDB}) end. %% Generate a clang compilation db entry for Src and Cmd @@ -394,7 +395,8 @@ get_port_spec(Config, OsType, {_Arch, Target, Sources, Opts}) -> LinkLang = case lists:any( fun(Src) -> compiler(filename:extension(Src)) == "$CXX" end, - SourceFiles) of + SourceFiles) + of true -> cxx; false -> cc end, From 31297b06f61ac24587c9705985b6ecdcb2b45ac5 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sun, 28 Aug 2016 19:34:54 +0200 Subject: [PATCH 64/89] rebar_utils: restore blank line --- src/rebar_utils.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index c3ebfe54..a5cc0ffd 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -94,6 +94,7 @@ is_arch(ArchRegex) -> nomatch -> false end. + %% %% REBAR_TARGET_ARCH, if used, should be set to the "standard" %% target string. That is a prefix for binutils tools. From e4144e9f9d783557860ab599c3b5a0fbf879e88d Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Tue, 30 Aug 2016 15:31:27 +0200 Subject: [PATCH 65/89] port_compiler: fix handling of multiple specs --- src/rebar_port_compiler.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rebar_port_compiler.erl b/src/rebar_port_compiler.erl index 7aa04ec7..9679c80e 100644 --- a/src/rebar_port_compiler.erl +++ b/src/rebar_port_compiler.erl @@ -216,15 +216,15 @@ replace_extension(File, OldExt, NewExt) -> %% compile_sources(Config, Specs, SharedEnv) -> - {Res, Db} = + {NewBins, Db} = lists:foldl( - fun(#spec{sources=Sources, type=Type, opts=Opts}, NewBins) -> + fun(#spec{sources=Sources, type=Type, opts=Opts}, Acc) -> Env = proplists:get_value(env, Opts, SharedEnv), - compile_each(Config, Sources, Type, Env, {NewBins, []}) - end, [], Specs), + compile_each(Config, Sources, Type, Env, Acc) + end, {[], []}, Specs), %% Rewrite clang compile commands database file only if something %% was compiled. - case Res of + case NewBins of [] -> ok; _ -> @@ -234,7 +234,7 @@ compile_sources(Config, Specs, SharedEnv) -> ok = io:fwrite(ClangDbFile, "]~n", []), ok = file:close(ClangDbFile) end, - Res. + NewBins. compile_each(_Config, [], _Type, _Env, {NewBins, CDB}) -> {lists:reverse(NewBins), lists:reverse(CDB)}; From defcc6f3177fbd0ba94686905656074fb808d68b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Wed, 31 Aug 2016 15:31:36 +0100 Subject: [PATCH 66/89] Prepare release 2.6.4 --- RELEASE-NOTES.md | 4 ++++ ebin/rebar.app | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 125c2815..dbb5402b 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,7 @@ +# 2.6.4 + +* rebar/619: [Fix cdb processing when a file is skipped](https://github.com/rebar/rebar/pull/619) + # 2.6.3 * rebar/610: [qc: lift experimental notice](https://github.com/rebar/rebar/pull/610) diff --git a/ebin/rebar.app b/ebin/rebar.app index 1ad483be..9449e1e4 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "2.6.3"}, + {vsn, "2.6.4"}, {modules, [ rebar, rebar_abnfc_compiler, rebar_app_utils, From b6634775f7561ee8bf31395ed5b0cae76f7c707c Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 16 Sep 2016 22:13:10 +0200 Subject: [PATCH 67/89] Remove superfluous space in unabbrev error msg --- src/rebar.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebar.erl b/src/rebar.erl index 2eb608d0..6f78fe66 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -593,7 +593,7 @@ unabbreviate_command_names([Command | Commands]) -> [FullCommand] -> [FullCommand | unabbreviate_command_names(Commands)]; Candidates -> - ?ABORT("Found more than one match for abbreviated command name " + ?ABORT("Found more than one match for abbreviated command name" " '~s',~nplease be more specific. Possible candidates:~n" " ~s~n", [Command, string:join(Candidates, ", ")]) From a8d17f9c15cb08402dd8638b4a1d174ee9b68892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Rasc=C3=A3o?= Date: Thu, 22 Sep 2016 11:48:09 +0100 Subject: [PATCH 68/89] Continuous integration on OTP 19.1 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e4a3149c..95898065 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ otp_release: - R14B03 - 17.0 - 18.0 + - 19.1 before_script: - hostname -f - cc -v From 6300009717067547bde7f5414d6e1f052bdbd922 Mon Sep 17 00:00:00 2001 From: Pete Higgins Date: Tue, 21 Feb 2017 11:31:24 -0800 Subject: [PATCH 69/89] Fix typo in eunit help. --- src/rebar_eunit.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index f4d7b76d..ebf76bcf 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -54,8 +54,8 @@ %% %%
  • %% tests="baz"- For every existing suite, run the first test whose -%% name starts with bar and, if no such test exists, run the test -%% whose name starts with bar in the suite's _tests module +%% name starts with baz and, if no such test exists, run the test +%% whose name starts with baz in the suite's _tests module %%
    • %% %% Additionally, for projects that have separate folders for the core From 59ea0a3f9cebaecf59af7b03e995e89a3c7ffdd1 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Mon, 15 May 2017 15:53:37 -0500 Subject: [PATCH 70/89] Properly skip apps with a .app.src.script file Any application using a .app.src.script file will now be properly skipped if requested. This check was missing the newer third case for the file extension. --- src/rebar_core.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 6cc8d385..747fe970 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -190,8 +190,8 @@ skip_or_process_dir(Dir, Command, Config, DirSet, CurrentCodePath, WouldCd); skip_or_process_dir(Dir, Command, Config, DirSet, CurrentCodePath, {_, File}=ModuleSet, WouldCd) -> - case lists:suffix(".app.src", File) - orelse lists:suffix(".app", File) of + case lists:suffix(".app", File) + orelse rebar_app_utils:is_app_src(File) of true -> %% .app or .app.src file, check if is_skipped_app skip_or_process_dir1(Dir, Command, Config, DirSet, CurrentCodePath, From 30002bd8bde2434c2cc2303e55a318f784bdce52 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 18 Aug 2017 18:52:24 +0000 Subject: [PATCH 71/89] travis-ci: test with Erlang 20.0.2 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 95898065..7af52589 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ otp_release: - 17.0 - 18.0 - 19.1 + - 20.0.2 before_script: - hostname -f - cc -v From 6ece7857bc80d8b8a413843b012bdf044e25ba25 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 18 Aug 2017 18:56:38 +0000 Subject: [PATCH 72/89] Fix #634 --- src/rebar_neotoma_compiler.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rebar_neotoma_compiler.erl b/src/rebar_neotoma_compiler.erl index f93d492e..e35dfc46 100644 --- a/src/rebar_neotoma_compiler.erl +++ b/src/rebar_neotoma_compiler.erl @@ -78,12 +78,18 @@ clean(Config, _AppFile) -> %% ============================================================================ info(help, compile) -> + info_help("Build Neotoma (*.peg) sources.~n"); +info(help, clean) -> + info_help("Delete *.peg build results"). + +info_help(Description) -> ?CONSOLE( - "Build Neotoma (*.peg) sources.~n" + "~s.~n" "~n" "Valid rebar.config options:~n" " ~p~n", [ + Description, {neotoma_opts, [{doc_root, "src"}, {out_dir, "src"}, {source_ext, ".peg"}, From b55483c88960747d3951190c908ba9f7e8ee1134 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 18 Aug 2017 19:10:49 +0000 Subject: [PATCH 73/89] Fix Erlang 20 export_all warnings Fix other warnings and whitespace errors as well. --- test/rebar_compiler_tests.erl | 10 ------ test/rebar_dia_tests.erl | 14 +++----- test/rebar_eunit_tests.erl | 8 ----- test/rebar_file_utils_tests.erl | 2 -- test/rebar_require_vsn_tests.erl | 2 -- test/rebar_utils_tests.erl | 2 -- test/rebar_xref_eunit.erl | 61 +++++++++++++++----------------- 7 files changed, 32 insertions(+), 67 deletions(-) diff --git a/test/rebar_compiler_tests.erl b/test/rebar_compiler_tests.erl index 104a7d88..2afbc92a 100644 --- a/test/rebar_compiler_tests.erl +++ b/test/rebar_compiler_tests.erl @@ -30,8 +30,6 @@ %% ------------------------------------------------------------------- -module(rebar_compiler_tests). --compile(export_all). - -include_lib("eunit/include/eunit.hrl"). %% Assuming this test is run inside the rebar 'eunit' @@ -167,19 +165,11 @@ prepare_rebar_script() -> ?TMP_DIR ++ "rebar.cmd") end. -rebar() -> - rebar([]). - rebar(Args) when is_list(Args) -> Out = os:cmd(filename:nativename("./rebar") ++ " " ++ Args), %% ?debugMsg("**** Begin"), ?debugMsg(Out), ?debugMsg("**** End"), Out. -assert_dirs_in(Name, [Dir|T]) -> - [{Name ++ " has directory: " ++ Dir, ?_assert(filelib:is_dir(Dir))} | - assert_dirs_in(Name, T)]; -assert_dirs_in(_, []) -> []. - assert_files_in(Name, [File|T]) -> [{Name ++ " has file: " ++ File, ?_assert(filelib:is_regular(File))} | assert_files_in(Name, T)]; diff --git a/test/rebar_dia_tests.erl b/test/rebar_dia_tests.erl index 556a552d..0561bbbe 100644 --- a/test/rebar_dia_tests.erl +++ b/test/rebar_dia_tests.erl @@ -2,8 +2,6 @@ %% ex: ts=4 sw=4 et -module(rebar_dia_tests). --compile([export_all]). - -include_lib("eunit/include/eunit.hrl"). -define(REBAR_SCRIPT, "../rebar"). @@ -12,7 +10,7 @@ dia_test_() -> case supported_otp_version() of - true -> + true -> {"Test the dia compiler", setup, fun() -> setup_project(), rebar("compile") end, fun teardown/1, @@ -22,7 +20,7 @@ dia_test_() -> {"Check include is created", ?_assert(filelib:is_dir("include") =:= true)}, - + {"Check dia/a.dia is compiled", ?_assert(string:str(RebarOut, "Compiled dia/a.dia") /= 0)}, @@ -62,7 +60,7 @@ dia_test_() -> supported_otp_version() -> Min = rebar_require_vsn:version_tuple(keep_going, "15", "configured"), - Otp = rebar_require_vsn:version_tuple(keep_going, + Otp = rebar_require_vsn:version_tuple(keep_going, erlang:system_info(otp_release), "OTP Release"), Otp >= Min. @@ -108,9 +106,5 @@ prepare_rebar_script() -> ?TMP_DIR ++ "rebar.cmd") end. -rebar() -> - rebar([]). - rebar(Args) when is_list(Args) -> - Out = os:cmd(filename:nativename("./rebar") ++ " " ++ Args), - Out. + os:cmd(filename:nativename("./rebar") ++ " " ++ Args). diff --git a/test/rebar_eunit_tests.erl b/test/rebar_eunit_tests.erl index d481daee..cf1e6960 100644 --- a/test/rebar_eunit_tests.erl +++ b/test/rebar_eunit_tests.erl @@ -30,8 +30,6 @@ %% ------------------------------------------------------------------- -module(rebar_eunit_tests). --compile(export_all). - -include_lib("eunit/include/eunit.hrl"). %% Assuming this test is run inside the rebar 'eunit' @@ -412,7 +410,6 @@ code_path_test_() -> -define(myapp_mymod_tests, ["-module(myapp_mymod_tests).\n", - "-compile([export_all]).\n", "-include_lib(\"eunit/include/eunit.hrl\").\n", "myfunc_test() -> ?assertMatch(ok, myapp_mymod:myfunc()).\n"]). @@ -426,7 +423,6 @@ code_path_test_() -> -define(myapp_mymod2_tests, ["-module(myapp_mymod2_tests).\n", - "-compile([export_all]).\n", "-include_lib(\"eunit/include/eunit.hrl\").\n", "myfunc2_test() -> ?assertMatch(ok, myapp_mymod2:myfunc2()).\n", "common_name_test() -> ?assert(true).\n"]). @@ -459,7 +455,6 @@ code_path_test_() -> -define(myapp_mymod_defined_in_mysuite_tests, ["-module(myapp_mymod_defined_in_mysuite_tests).\n", - "-compile([export_all]).\n", "-include_lib(\"eunit/include/eunit.hrl\").\n", "myfunc_test() -> ?assertMatch(ok, myapp_mymod:myfunc()).\n"]). @@ -537,9 +532,6 @@ prepare_rebar_script() -> ?TMP_DIR ++ "rebar.cmd") end. -rebar() -> - rebar([]). - rebar(Args) when is_list(Args) -> Out = os:cmd(filename:nativename("./rebar") ++ " " ++ Args), %% ?debugMsg("**** Begin"), ?debugMsg(Out), ?debugMsg("**** End"), diff --git a/test/rebar_file_utils_tests.erl b/test/rebar_file_utils_tests.erl index c9b4192c..ca5edf05 100644 --- a/test/rebar_file_utils_tests.erl +++ b/test/rebar_file_utils_tests.erl @@ -30,8 +30,6 @@ %% ------------------------------------------------------------------- -module(rebar_file_utils_tests). --compile(export_all). - -include_lib("eunit/include/eunit.hrl"). -define(TMP_DIR, "tmp_file_utils"). diff --git a/test/rebar_require_vsn_tests.erl b/test/rebar_require_vsn_tests.erl index a934e8e7..43333d33 100644 --- a/test/rebar_require_vsn_tests.erl +++ b/test/rebar_require_vsn_tests.erl @@ -2,8 +2,6 @@ %% ex: ts=4 sw=4 et -module(rebar_require_vsn_tests). --compile(export_all). - -include_lib("eunit/include/eunit.hrl"). version_tuple_test_() -> diff --git a/test/rebar_utils_tests.erl b/test/rebar_utils_tests.erl index 9bc393ac..81f5c92a 100644 --- a/test/rebar_utils_tests.erl +++ b/test/rebar_utils_tests.erl @@ -30,8 +30,6 @@ %% ------------------------------------------------------------------- -module(rebar_utils_tests). --compile(export_all). - -include_lib("eunit/include/eunit.hrl"). expand_env_variable_test_() -> diff --git a/test/rebar_xref_eunit.erl b/test/rebar_xref_eunit.erl index f32ea46b..13d8a5e4 100644 --- a/test/rebar_xref_eunit.erl +++ b/test/rebar_xref_eunit.erl @@ -2,8 +2,6 @@ %% ex: ts=4 sw=4 et -module(rebar_xref_eunit). --compile(export_all). - -include_lib("eunit/include/eunit.hrl"). -define(REBAR_SCRIPT, "../rebar"). @@ -16,31 +14,31 @@ xref_test_() -> fun teardown/1, fun(RebarOut) -> [ - {"Undefined function", ?_assert(string:str(RebarOut, + {"Undefined function", ?_assert(string:str(RebarOut, "myapp_somemod:notavailable/1 is undefined function") =/= 0)}, - {"Undefined function call", ?_assert(string:str(RebarOut, + {"Undefined function call", ?_assert(string:str(RebarOut, "myapp_othermod:somefunc/0 calls undefined function myapp_somemod:notavailable/1") =/= 0)}, - {"Deprecated function", ?_assert(string:str(RebarOut, + {"Deprecated function", ?_assert(string:str(RebarOut, "myapp_mymod:fdeprecated/0 is deprecated function") =/= 0)}, - {"Deprecated function call", ?_assert(string:str(RebarOut, + {"Deprecated function call", ?_assert(string:str(RebarOut, "myapp_othermod:somefunc/0 calls deprecated function myapp_mymod:fdeprecated/0") =/= 0)}, - {"Unused local", ?_assert(string:str(RebarOut, + {"Unused local", ?_assert(string:str(RebarOut, "myapp_mymod:localfunc2/0 is unused local function") =/= 0)}, - {"Unused export 1", ?_assert(string:str(RebarOut, + {"Unused export 1", ?_assert(string:str(RebarOut, "myapp_behaviour1:behaviour_info/1 is unused export") =/= 0)}, - {"Unused export 2", ?_assert(string:str(RebarOut, + {"Unused export 2", ?_assert(string:str(RebarOut, "myapp_behaviour2:behaviour_info/1 is unused export") =/= 0)}, - {"Unused export 3", ?_assert(string:str(RebarOut, + {"Unused export 3", ?_assert(string:str(RebarOut, "myapp_mymod:other2/1 is unused export") =/= 0)}, - {"Unused export 4", ?_assert(string:str(RebarOut, + {"Unused export 4", ?_assert(string:str(RebarOut, "myapp_othermod:somefunc/0 is unused export") =/= 0)}, - {"Suppressed behaviour export 1", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 1", ?_assert(string:str(RebarOut, "myapp_mymod:bh1_a/1 is unused export") =:= 0)}, - {"Suppressed behaviour export 2", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 2", ?_assert(string:str(RebarOut, "myapp_mymod:bh1_b/1 is unused export") =:= 0)}, - {"Suppressed behaviour export 3", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 3", ?_assert(string:str(RebarOut, "myapp_mymod:bh2_a/1 is unused export") =:= 0)}, - {"Suppressed behaviour export 4", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 4", ?_assert(string:str(RebarOut, "myapp_mymod:bh2_b/1 is unused export") =:= 0)} ] end}. @@ -51,31 +49,31 @@ xref_ignore_test_() -> fun teardown/1, fun(RebarOut) -> [ - {"Undefined function can not be suppressed.", ?_assert(string:str(RebarOut, + {"Undefined function can not be suppressed.", ?_assert(string:str(RebarOut, "myapp_somemod:notavailable/1 is undefined function") =/= 0)}, - {"Supppressed undefined function call", ?_assert(string:str(RebarOut, + {"Supppressed undefined function call", ?_assert(string:str(RebarOut, "myapp_othermod:somefunc/0 calls undefined function myapp_somemod:notavailable/1") =:= 0)}, - {"Supppressed deprecated function", ?_assert(string:str(RebarOut, + {"Supppressed deprecated function", ?_assert(string:str(RebarOut, "myapp_mymod:fdeprecated/0 is deprecated function") =:= 0)}, - {"Supppressed deprecated function call", ?_assert(string:str(RebarOut, + {"Supppressed deprecated function call", ?_assert(string:str(RebarOut, "myapp_othermod:somefunc/0 calls deprecated function myapp_mymod:fdeprecated/0") =:= 0)}, - {"Supppressed unused local", ?_assert(string:str(RebarOut, + {"Supppressed unused local", ?_assert(string:str(RebarOut, "myapp_mymod:localfunc2/0 is unused local function") =:= 0)}, - {"Supppressed unused export 1", ?_assert(string:str(RebarOut, + {"Supppressed unused export 1", ?_assert(string:str(RebarOut, "myapp_behaviour1:behaviour_info/1 is unused export") =:= 0)}, - {"Supppressed unused export 2", ?_assert(string:str(RebarOut, + {"Supppressed unused export 2", ?_assert(string:str(RebarOut, "myapp_behaviour2:behaviour_info/1 is unused export") =:= 0)}, - {"Supppressed unused export 3", ?_assert(string:str(RebarOut, + {"Supppressed unused export 3", ?_assert(string:str(RebarOut, "myapp_mymod:other2/1 is unused export") =:= 0)}, - {"Supppressed unused export 4", ?_assert(string:str(RebarOut, + {"Supppressed unused export 4", ?_assert(string:str(RebarOut, "myapp_othermod:somefunc/0 is unused export") =:= 0)}, - {"Suppressed behaviour export 1", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 1", ?_assert(string:str(RebarOut, "myapp_mymod:bh1_a/1 is unused export") =:= 0)}, - {"Suppressed behaviour export 2", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 2", ?_assert(string:str(RebarOut, "myapp_mymod:bh1_b/1 is unused export") =:= 0)}, - {"Suppressed behaviour export 3", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 3", ?_assert(string:str(RebarOut, "myapp_mymod:bh2_a/1 is unused export") =:= 0)}, - {"Suppressed behaviour export 4", ?_assert(string:str(RebarOut, + {"Suppressed behaviour export 4", ?_assert(string:str(RebarOut, "myapp_mymod:bh2_b/1 is unused export") =:= 0)} ] @@ -116,7 +114,7 @@ xref_ignore_test_() -> "bh2_a(A) -> localfunc1(bh2_a, A).\n", "bh2_b(A) -> localfunc1(bh2_b, A).\n", "other1(A) -> localfunc1(other1, A).\n", % regular exported functions - "other2(A) -> localfunc1(other2, A).\n", + "other2(A) -> localfunc1(other2, A).\n", "localfunc1(A, B) -> {A, B}.\n", % used local "localfunc2() -> ok.\n", % unused local "fdeprecated() -> ok.\n" % deprecated function @@ -155,7 +153,7 @@ setup_environment() -> prepare_project() -> setup_environment(), rebar("create-app appid=myapp"), - ok = file:make_dir("ebin"). + ok = file:make_dir("ebin"). setup_project(ignore_xref) -> prepare_project(), @@ -196,9 +194,6 @@ prepare_rebar_script() -> ?TMP_DIR ++ "rebar.cmd") end. -rebar() -> - rebar([]). - rebar(Args) when is_list(Args) -> Out = os:cmd(filename:nativename("./rebar") ++ " " ++ Args), %% ?debugMsg("**** Begin"), ?debugMsg(Out), ?debugMsg("**** End"), From 2a7774285b2973f085eb0a36647be2a64178673f Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 18 Aug 2017 20:22:02 +0000 Subject: [PATCH 74/89] rebar.config: access retest repo via https --- rebar.config.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.script b/rebar.config.script index eacd510b..67389b00 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,7 +1,7 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 ft=erlang et -ExtraDeps = [{retest, ".*", {git, "git://github.com/rebar/retest.git", +ExtraDeps = [{retest, ".*", {git, "https://github.com/rebar/retest", {tag, "1.1.0"}}}], case os:getenv("REBAR_EXTRA_DEPS") of From bb79416b6ef536c1b64876d8c9fe5e596dd1320a Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 18 Aug 2017 19:01:54 +0000 Subject: [PATCH 75/89] Drop crypto app reliance By doing this, we fix two issues at once. First, there's no need to have crypto available anymore. While not having crypto in your Erlang installation is a questionable packaging decision, it tends to happen in the wild with users installing Erlang and missing crypto. Sometimes this is not due to a distro's package but users building Erlang without the needed OpenSSL dependencies. Second, this resolves the Erlang 20 rng deprecation warnings. --- ebin/rebar.app | 1 - rebar.config | 3 ++- src/rebar.erl | 6 ------ src/rebar_ct.erl | 2 +- src/rebar_eunit.erl | 2 +- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/ebin/rebar.app b/ebin/rebar.app index 9449e1e4..b4c8b5a1 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -55,7 +55,6 @@ stdlib, sasl, compiler, - crypto, syntax_tools, tools, eunit, diff --git a/rebar.config b/rebar.config index eda5a2c8..6765415c 100644 --- a/rebar.config +++ b/rebar.config @@ -32,7 +32,8 @@ - (\"diameter_dict_util\":\"parse\"/\"2\") - (\"erlang\":\"timestamp\"/\"0\") - (\"rebar_rnd\":\"seed\"/\"1\") - - (\"rebar_rnd\":\"uniform\"/\"0\"))", + - (\"rebar_rnd\":\"uniform\"/\"0\") + - (\"rebar_rnd\":\"uniform\"/\"1\"))", []}]}. {dialyzer, diff --git a/src/rebar.erl b/src/rebar.erl index 6f78fe66..fa7d0608 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -208,12 +208,6 @@ profile(_Config, _Commands, Profiler) -> ?ABORT("Unsupported profiler: ~s~n", [Profiler]). run_aux(BaseConfig, Commands) -> - %% Make sure crypto is running - case crypto:start() of - ok -> ok; - {error,{already_started,crypto}} -> ok - end, - %% Make sure memoization server is running case rmemo:start() of {ok, _} -> ok; diff --git a/src/rebar_ct.erl b/src/rebar_ct.erl index b27f6617..7c0f9d97 100644 --- a/src/rebar_ct.erl +++ b/src/rebar_ct.erl @@ -288,7 +288,7 @@ search_ct_specs_from(Cwd, TestDir, Config) -> build_name(Config) -> %% generate a unique name for our test node, we want %% to make sure the odds of name clashing are low - Random = integer_to_list(crypto:rand_uniform(0, 10000)), + Random = integer_to_list(rebar_rnd:uniform(10000)), case rebar_config:get_local(Config, ct_use_short_names, false) of true -> "-sname test" ++ Random; false -> " -name test" ++ Random ++ "@" ++ net_adm:localhost() diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index ebf76bcf..538f88ba 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -279,7 +279,7 @@ randomize_suites(Config, Modules) -> undefined -> Modules; "true" -> - Seed = crypto:rand_uniform(1, 65535), + Seed = rebar_rnd:uniform(65535), randomize_suites1(Modules, Seed); String -> try list_to_integer(String) of From 13a865edcaa6959410747f805208a09781482e7b Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Fri, 18 Aug 2017 21:02:55 +0000 Subject: [PATCH 76/89] Improve pseudo-unique ct_run node name Replacing crypto:rand_uniform(0, 10000) with rebar_rnd:uniform(10000) (which is either rand:uniform or random:uniform dependent on what otp version it runs on) fails in rebar_ct. One might assume it's because the old call requests a number >= 0 while the new call has no way to ask for anything but >= 1. However, the way the value is used doesn't rely on it to include 0, so the bug lies elsewhere. Actually, generating a random number to choose a pseudo-unique ct_run node name isn't a reliable method to avoid naming clashes. I cannot reproduce it locally and also cannot see the actual error in the CI logs. Mirroring what rebar_eunit does and using a bigger rng range seems to be enough to avoid node name clashes on Travis-CI. --- src/rebar_ct.erl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/rebar_ct.erl b/src/rebar_ct.erl index 7c0f9d97..892ae036 100644 --- a/src/rebar_ct.erl +++ b/src/rebar_ct.erl @@ -237,6 +237,7 @@ make_cmd(TestDir, RawLogDir, Config) -> CodeDirs = [io_lib:format("\"~s\"", [Dir]) || Dir <- [EbinDir|NonLibCodeDirs]], CodePathString = string:join(CodeDirs, " "), + _ = rebar_rnd:seed({55, seconds(), 7331}), Cmd = case get_ct_specs(Config, search_ct_specs_from(Cwd, TestDir, Config)) of undefined -> ?FMT("~s" @@ -285,13 +286,18 @@ search_ct_specs_from(Cwd, TestDir, Config) -> Cwd end. +seconds() -> + calendar:datetime_to_gregorian_seconds(calendar:universal_time()). + build_name(Config) -> %% generate a unique name for our test node, we want %% to make sure the odds of name clashing are low - Random = integer_to_list(rebar_rnd:uniform(10000)), + Secs = integer_to_list(seconds()), + Random = integer_to_list(rebar_rnd:uniform(1000000)), + PseudoUnique = Random ++ "N" ++ Secs, case rebar_config:get_local(Config, ct_use_short_names, false) of - true -> "-sname test" ++ Random; - false -> " -name test" ++ Random ++ "@" ++ net_adm:localhost() + true -> "-sname test" ++ PseudoUnique; + false -> " -name test" ++ PseudoUnique ++ "@" ++ net_adm:localhost() end. get_extra_params(Config) -> From b65d3db2e0ccd313fa62c02f8759185567caee09 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sat, 19 Aug 2017 16:34:23 +0000 Subject: [PATCH 77/89] Fix ctsuite template - Fix and test the Erlang code. - Do not export_all. --- priv/templates/ctsuite.erl | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/priv/templates/ctsuite.erl b/priv/templates/ctsuite.erl index 33a8fac9..bed2ae63 100644 --- a/priv/templates/ctsuite.erl +++ b/priv/templates/ctsuite.erl @@ -3,7 +3,20 @@ -module({{testmod}}_SUITE). -include_lib("common_test/include/ct.hrl"). --compile(export_all). +-export([ suite/0 + , groups/0 + , all/0 + , init_per_suite/1 + , end_per_suite/1 + , init_per_group/2 + , end_per_group/2 + , init_per_testcase/2 + , end_per_testcase/2 + ]). + +-export([ test_{{testmod}}/0 + , test_{{testmod}}/1 + ]). %%-------------------------------------------------------------------- %% Function: suite() -> Info @@ -57,12 +70,16 @@ groups() -> []. %% NB: By default, we export all 1-arity user defined functions %%-------------------------------------------------------------------- all() -> - [ {exports, Functions} | _ ] = ?MODULE:module_info(), + { exports, Functions } = lists:keyfind(exports, 1, ?MODULE:module_info()), [ FName || {FName, _} <- lists:filter( fun ({module_info,_}) -> false; ({all,_}) -> false; ({init_per_suite,1}) -> false; ({end_per_suite,1}) -> false; + ({init_per_group,2}) -> false; + ({end_per_group,2}) -> false; + ({init_per_testcase,2}) -> false; + ({end_per_testcase,2}) -> false; ({_,1}) -> true; ({_,_}) -> false end, Functions)]. @@ -108,7 +125,7 @@ end_per_suite(_Config) -> %% %% Description: Initialization before each test case group. %%-------------------------------------------------------------------- -init_per_group(_group, Config) -> +init_per_group(_GroupName, Config) -> Config. %%-------------------------------------------------------------------- @@ -122,7 +139,7 @@ init_per_group(_group, Config) -> %% %% Description: Cleanup after each test case group. %%-------------------------------------------------------------------- -end_per_group(_group, Config) -> +end_per_group(_GroupName, Config) -> Config. %%-------------------------------------------------------------------- @@ -141,7 +158,7 @@ end_per_group(_group, Config) -> %% Note: This function is free to add any key/value pairs to the Config %% variable, but should NOT alter/remove any existing entries. %%-------------------------------------------------------------------- -init_per_testcase(TestCase, Config) -> +init_per_testcase(_TestCase, Config) -> Config. %%-------------------------------------------------------------------- @@ -157,7 +174,7 @@ init_per_testcase(TestCase, Config) -> %% %% Description: Cleanup after each test case. %%-------------------------------------------------------------------- -end_per_testcase(TestCase, Config) -> +end_per_testcase(_TestCase, Config) -> Config. test_{{testmod}}() -> From eb2512ff3fcf509368bbb9a76b906b350c2c4eb2 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sat, 19 Aug 2017 16:47:53 +0000 Subject: [PATCH 78/89] inttest/*: do not export_all --- inttest/app_src/app_src.erl | 2 +- inttest/app_src/app_src_rt.erl | 2 +- inttest/app_src_script/app_src_script.erl | 2 +- inttest/app_src_script/app_src_script_rt.erl | 2 +- inttest/app_src_script_2/app_src_script_2.erl | 2 +- inttest/app_src_script_2/app_src_script_2_rt.erl | 2 +- inttest/appup_src/appup_src_rt.erl | 2 +- inttest/appup_src/src/appup_src.erl | 2 +- inttest/appup_src_2/appup_src_2_rt.erl | 2 +- inttest/appup_src_2/appup_src_rt_2.erl | 2 +- inttest/appup_src_2/src/appup_src.erl | 2 +- inttest/appup_src_script/appup_src_script_rt.erl | 2 +- inttest/appup_src_script/src/appup_src.erl | 2 +- inttest/bug_5_rt.erl | 2 +- inttest/ct1/ct1_rt.erl | 2 +- inttest/ct1/test_SUITE.erl | 2 +- inttest/ct2/ct2_rt.erl | 2 +- inttest/ct2/foo_SUITE.erl | 2 +- inttest/ct3/ct3_rt.erl | 2 +- inttest/ct3/test_SUITE.erl | 2 +- inttest/ct4/ct4_rt.erl | 2 +- inttest/ct4/foo_SUITE.erl | 2 +- inttest/ct_cover/ct_cover_rt.erl | 2 +- inttest/ct_cover/mock/dummy/itest/test_SUITE.erl | 2 +- inttest/ct_cover/test_SUITE.erl | 2 +- inttest/ct_make_fails/ct_make_fails_rt.erl | 2 +- inttest/ct_make_fails/test_SUITE.erl | 2 +- inttest/ct_test_fails/ct_test_fails_rt.erl | 11 +++++------ inttest/ct_test_fails/test2_SUITE.erl | 2 +- inttest/ct_test_fails/test_SUITE.erl | 2 +- inttest/depplugins/depplugins_rt.erl | 2 +- inttest/inttest_utils.erl | 5 ++++- inttest/neotoma1/neotoma_src_rt.erl | 2 +- inttest/rebar3_deps1/a.erl | 2 +- inttest/rebar3_deps1/rebar3_deps1_rt.erl | 2 +- inttest/rgen1/rgen1_rt.erl | 2 +- inttest/t_custom_config/custom_config.erl | 2 +- inttest/t_custom_config/t_custom_config_rt.erl | 2 +- inttest/tdeps1/a.erl | 2 +- inttest/tdeps1/tdeps1_rt.erl | 2 +- inttest/tdeps2/tdeps2_rt.erl | 2 +- inttest/tdeps3/tdeps3_rt.erl | 2 +- inttest/tdeps_prefer/tdeps_prefer_rt.erl | 2 +- inttest/tdeps_update/tdeps_update_rt.erl | 2 +- inttest/thooks/fish.erl | 2 +- inttest/thooks/thooks_rt.erl | 2 +- inttest/tplugins/bad_plugin.erl | 3 ++- inttest/tplugins/fish.erl | 2 +- inttest/tplugins/test_plugin.erl | 3 ++- inttest/tplugins/tplugins_rt.erl | 2 +- inttest/vsn_cache/vsn_cache_rt.erl | 2 +- 51 files changed, 60 insertions(+), 56 deletions(-) diff --git a/inttest/app_src/app_src.erl b/inttest/app_src/app_src.erl index dec73079..a6c53d13 100644 --- a/inttest/app_src/app_src.erl +++ b/inttest/app_src/app_src.erl @@ -1,6 +1,6 @@ -module(app_src). --compile(export_all). +-export([test/0]). test() -> ok. diff --git a/inttest/app_src/app_src_rt.erl b/inttest/app_src/app_src_rt.erl index 2b5a87e9..c8240d2f 100644 --- a/inttest/app_src/app_src_rt.erl +++ b/inttest/app_src/app_src_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(app_src_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/app_src_script/app_src_script.erl b/inttest/app_src_script/app_src_script.erl index 6372c92e..3f2a19b9 100644 --- a/inttest/app_src_script/app_src_script.erl +++ b/inttest/app_src_script/app_src_script.erl @@ -1,6 +1,6 @@ -module(app_src_script). --compile(export_all). +-export([test/0]). test() -> ok. diff --git a/inttest/app_src_script/app_src_script_rt.erl b/inttest/app_src_script/app_src_script_rt.erl index c0de128e..377f929e 100644 --- a/inttest/app_src_script/app_src_script_rt.erl +++ b/inttest/app_src_script/app_src_script_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(app_src_script_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/app_src_script_2/app_src_script_2.erl b/inttest/app_src_script_2/app_src_script_2.erl index 26b83642..88eee51b 100644 --- a/inttest/app_src_script_2/app_src_script_2.erl +++ b/inttest/app_src_script_2/app_src_script_2.erl @@ -1,6 +1,6 @@ -module(app_src_script_2). --compile(export_all). +-export([test/0]). test() -> ok. diff --git a/inttest/app_src_script_2/app_src_script_2_rt.erl b/inttest/app_src_script_2/app_src_script_2_rt.erl index 613e3f9e..78cd7616 100644 --- a/inttest/app_src_script_2/app_src_script_2_rt.erl +++ b/inttest/app_src_script_2/app_src_script_2_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(app_src_script_2_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/appup_src/appup_src_rt.erl b/inttest/appup_src/appup_src_rt.erl index 7530aa49..01319729 100644 --- a/inttest/appup_src/appup_src_rt.erl +++ b/inttest/appup_src/appup_src_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(appup_src_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/appup_src/src/appup_src.erl b/inttest/appup_src/src/appup_src.erl index 6f4d0d71..26b18125 100644 --- a/inttest/appup_src/src/appup_src.erl +++ b/inttest/appup_src/src/appup_src.erl @@ -1,6 +1,6 @@ -module(appup_src). --compile(export_all). +-export([test/0]). test() -> ok. diff --git a/inttest/appup_src_2/appup_src_2_rt.erl b/inttest/appup_src_2/appup_src_2_rt.erl index 99e62487..4b621b10 100644 --- a/inttest/appup_src_2/appup_src_2_rt.erl +++ b/inttest/appup_src_2/appup_src_2_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(appup_src_2_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/appup_src_2/appup_src_rt_2.erl b/inttest/appup_src_2/appup_src_rt_2.erl index fff6f627..e855a969 100644 --- a/inttest/appup_src_2/appup_src_rt_2.erl +++ b/inttest/appup_src_2/appup_src_rt_2.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(appup_src_rt_2). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/appup_src_2/src/appup_src.erl b/inttest/appup_src_2/src/appup_src.erl index 6f4d0d71..26b18125 100644 --- a/inttest/appup_src_2/src/appup_src.erl +++ b/inttest/appup_src_2/src/appup_src.erl @@ -1,6 +1,6 @@ -module(appup_src). --compile(export_all). +-export([test/0]). test() -> ok. diff --git a/inttest/appup_src_script/appup_src_script_rt.erl b/inttest/appup_src_script/appup_src_script_rt.erl index 1aeb680a..5dab165f 100644 --- a/inttest/appup_src_script/appup_src_script_rt.erl +++ b/inttest/appup_src_script/appup_src_script_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(appup_src_script_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/appup_src_script/src/appup_src.erl b/inttest/appup_src_script/src/appup_src.erl index 6f4d0d71..26b18125 100644 --- a/inttest/appup_src_script/src/appup_src.erl +++ b/inttest/appup_src_script/src/appup_src.erl @@ -1,6 +1,6 @@ -module(appup_src). --compile(export_all). +-export([test/0]). test() -> ok. diff --git a/inttest/bug_5_rt.erl b/inttest/bug_5_rt.erl index f198d40d..7f00e4ab 100644 --- a/inttest/bug_5_rt.erl +++ b/inttest/bug_5_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(bug_5_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/ct1/ct1_rt.erl b/inttest/ct1/ct1_rt.erl index 03b4ed8d..97d77f90 100644 --- a/inttest/ct1/ct1_rt.erl +++ b/inttest/ct1/ct1_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(ct1_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/ct1/test_SUITE.erl b/inttest/ct1/test_SUITE.erl index e8a2bb80..3d86a6ad 100644 --- a/inttest/ct1/test_SUITE.erl +++ b/inttest/ct1/test_SUITE.erl @@ -1,6 +1,6 @@ -module(test_SUITE). --compile(export_all). +-export([all/0, simple_test/1, app_config_file_test/1]). -include_lib("ct.hrl"). diff --git a/inttest/ct2/ct2_rt.erl b/inttest/ct2/ct2_rt.erl index b3138d40..06d7abbd 100644 --- a/inttest/ct2/ct2_rt.erl +++ b/inttest/ct2/ct2_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(ct2_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/ct2/foo_SUITE.erl b/inttest/ct2/foo_SUITE.erl index d03aedf0..be089c78 100644 --- a/inttest/ct2/foo_SUITE.erl +++ b/inttest/ct2/foo_SUITE.erl @@ -2,7 +2,7 @@ -include_lib("common_test/include/ct.hrl"). --compile(export_all). +-export([all/0, foo/1]). all() -> [foo]. diff --git a/inttest/ct3/ct3_rt.erl b/inttest/ct3/ct3_rt.erl index b5739db5..32ded4cf 100644 --- a/inttest/ct3/ct3_rt.erl +++ b/inttest/ct3/ct3_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(ct3_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/ct3/test_SUITE.erl b/inttest/ct3/test_SUITE.erl index efd930d6..610c14cf 100644 --- a/inttest/ct3/test_SUITE.erl +++ b/inttest/ct3/test_SUITE.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(test_SUITE). --compile(export_all). +-export([all/0, simple_test/1, app_config_file_test/1, check_path/1]). -include_lib("ct.hrl"). diff --git a/inttest/ct4/ct4_rt.erl b/inttest/ct4/ct4_rt.erl index bc010e86..b393bc74 100644 --- a/inttest/ct4/ct4_rt.erl +++ b/inttest/ct4/ct4_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(ct4_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/ct4/foo_SUITE.erl b/inttest/ct4/foo_SUITE.erl index fb4f56a9..931ca50a 100644 --- a/inttest/ct4/foo_SUITE.erl +++ b/inttest/ct4/foo_SUITE.erl @@ -2,7 +2,7 @@ -include_lib("common_test/include/ct.hrl"). --compile(export_all). +-export([all/0, simple/1]). all() -> [simple]. diff --git a/inttest/ct_cover/ct_cover_rt.erl b/inttest/ct_cover/ct_cover_rt.erl index 361d85bd..8bff9df2 100644 --- a/inttest/ct_cover/ct_cover_rt.erl +++ b/inttest/ct_cover/ct_cover_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(ct_cover_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/ct_cover/mock/dummy/itest/test_SUITE.erl b/inttest/ct_cover/mock/dummy/itest/test_SUITE.erl index 0d9a065d..25dde6a4 100644 --- a/inttest/ct_cover/mock/dummy/itest/test_SUITE.erl +++ b/inttest/ct_cover/mock/dummy/itest/test_SUITE.erl @@ -1,6 +1,6 @@ -module(test_SUITE). --compile(export_all). +-export([all/0, simple_test/1, app_config_file_test/1]). -include_lib("ct.hrl"). diff --git a/inttest/ct_cover/test_SUITE.erl b/inttest/ct_cover/test_SUITE.erl index e8a2bb80..3d86a6ad 100644 --- a/inttest/ct_cover/test_SUITE.erl +++ b/inttest/ct_cover/test_SUITE.erl @@ -1,6 +1,6 @@ -module(test_SUITE). --compile(export_all). +-export([all/0, simple_test/1, app_config_file_test/1]). -include_lib("ct.hrl"). diff --git a/inttest/ct_make_fails/ct_make_fails_rt.erl b/inttest/ct_make_fails/ct_make_fails_rt.erl index a7959a7d..e2abb9a7 100644 --- a/inttest/ct_make_fails/ct_make_fails_rt.erl +++ b/inttest/ct_make_fails/ct_make_fails_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(ct_make_fails_rt). --compile(export_all). +-export([files/0, run/1]). files() -> diff --git a/inttest/ct_make_fails/test_SUITE.erl b/inttest/ct_make_fails/test_SUITE.erl index 343aa5aa..068e9267 100644 --- a/inttest/ct_make_fails/test_SUITE.erl +++ b/inttest/ct_make_fails/test_SUITE.erl @@ -1,6 +1,6 @@ -module(test_SUITE). --compile(export_all). +-export([all/0, simple_test/1, app_config_file_test/1]). -include_lib("ct.hrl"). diff --git a/inttest/ct_test_fails/ct_test_fails_rt.erl b/inttest/ct_test_fails/ct_test_fails_rt.erl index a556891f..69ee3821 100644 --- a/inttest/ct_test_fails/ct_test_fails_rt.erl +++ b/inttest/ct_test_fails/ct_test_fails_rt.erl @@ -2,8 +2,7 @@ %% ex: ts=4 sw=4 et -module(ct_test_fails_rt). --compile(export_all). - +-export([files/0, run/1]). files() -> [{create, "ebin/a1.app", app(a1)}, @@ -14,11 +13,11 @@ files() -> {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}, {copy, "test2_SUITE.erl", "itest/test2_SUITE.erl"}]. -run(Dir) -> +run(_Dir) -> ok = case catch retest:sh("./rebar compile ct -v 3") of - {error, {stopped, _}} -> ok; - _ -> expected_to_fail - end. + {error, {stopped, _}} -> ok; + _ -> expected_to_fail + end. %% %% Generate the contents of a simple .app file diff --git a/inttest/ct_test_fails/test2_SUITE.erl b/inttest/ct_test_fails/test2_SUITE.erl index 4b9299fe..7db1d229 100644 --- a/inttest/ct_test_fails/test2_SUITE.erl +++ b/inttest/ct_test_fails/test2_SUITE.erl @@ -1,6 +1,6 @@ -module(test2_SUITE). --compile(export_all). +-export([all/0, simple_test/1]). all() -> [simple_test]. diff --git a/inttest/ct_test_fails/test_SUITE.erl b/inttest/ct_test_fails/test_SUITE.erl index 124fbe8a..047631c4 100644 --- a/inttest/ct_test_fails/test_SUITE.erl +++ b/inttest/ct_test_fails/test_SUITE.erl @@ -1,6 +1,6 @@ -module(test_SUITE). --compile(export_all). +-export([all/0, simple_test/1]). all() -> [simple_test]. diff --git a/inttest/depplugins/depplugins_rt.erl b/inttest/depplugins/depplugins_rt.erl index fc72dec2..c2cb76ae 100644 --- a/inttest/depplugins/depplugins_rt.erl +++ b/inttest/depplugins/depplugins_rt.erl @@ -17,7 +17,7 @@ %%% -module(depplugins_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/inttest_utils.erl b/inttest/inttest_utils.erl index d896f4d3..addac5f5 100644 --- a/inttest/inttest_utils.erl +++ b/inttest/inttest_utils.erl @@ -2,7 +2,10 @@ %% ex: ts=4 sw=4 et -module(inttest_utils). --compile(export_all). +-export([ rebar_setup/1 + , rebar_setup/2 + , rebar_setup/0 + ]). rebar_setup({win32, nt}, Dir) -> [{copy, filename:join(Dir, "rebar.cmd"), "rebar.cmd"}]; diff --git a/inttest/neotoma1/neotoma_src_rt.erl b/inttest/neotoma1/neotoma_src_rt.erl index 6f7c6fff..568bc314 100644 --- a/inttest/neotoma1/neotoma_src_rt.erl +++ b/inttest/neotoma1/neotoma_src_rt.erl @@ -26,7 +26,7 @@ %% ------------------------------------------------------------------- -module(neotoma_src_rt). --compile(export_all). +-export([files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("deps/retest/include/retest.hrl"). diff --git a/inttest/rebar3_deps1/a.erl b/inttest/rebar3_deps1/a.erl index 835522ac..897b963c 100644 --- a/inttest/rebar3_deps1/a.erl +++ b/inttest/rebar3_deps1/a.erl @@ -1,6 +1,6 @@ -module(a). --compile(export_all). +-export([hello/0]). -include_lib("b/include/b.hrl"). diff --git a/inttest/rebar3_deps1/rebar3_deps1_rt.erl b/inttest/rebar3_deps1/rebar3_deps1_rt.erl index e84b6ba3..92a1628b 100644 --- a/inttest/rebar3_deps1/rebar3_deps1_rt.erl +++ b/inttest/rebar3_deps1/rebar3_deps1_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(rebar3_deps1_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/rgen1/rgen1_rt.erl b/inttest/rgen1/rgen1_rt.erl index 5847f4a1..7641ca4e 100644 --- a/inttest/rgen1/rgen1_rt.erl +++ b/inttest/rgen1/rgen1_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(rgen1_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). %% Exercise release generation w/ templating diff --git a/inttest/t_custom_config/custom_config.erl b/inttest/t_custom_config/custom_config.erl index 86562014..de964b4c 100644 --- a/inttest/t_custom_config/custom_config.erl +++ b/inttest/t_custom_config/custom_config.erl @@ -1,6 +1,6 @@ -module(custom_config). --compile(export_all). +-export([test/0]). test() -> ok. diff --git a/inttest/t_custom_config/t_custom_config_rt.erl b/inttest/t_custom_config/t_custom_config_rt.erl index 1f21927c..bda229e7 100644 --- a/inttest/t_custom_config/t_custom_config_rt.erl +++ b/inttest/t_custom_config/t_custom_config_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(t_custom_config_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/tdeps1/a.erl b/inttest/tdeps1/a.erl index 835522ac..897b963c 100644 --- a/inttest/tdeps1/a.erl +++ b/inttest/tdeps1/a.erl @@ -1,6 +1,6 @@ -module(a). --compile(export_all). +-export([hello/0]). -include_lib("b/include/b.hrl"). diff --git a/inttest/tdeps1/tdeps1_rt.erl b/inttest/tdeps1/tdeps1_rt.erl index 849403b5..b291b544 100644 --- a/inttest/tdeps1/tdeps1_rt.erl +++ b/inttest/tdeps1/tdeps1_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(tdeps1_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/tdeps2/tdeps2_rt.erl b/inttest/tdeps2/tdeps2_rt.erl index a5665ad2..33a8c15a 100644 --- a/inttest/tdeps2/tdeps2_rt.erl +++ b/inttest/tdeps2/tdeps2_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(tdeps2_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/tdeps3/tdeps3_rt.erl b/inttest/tdeps3/tdeps3_rt.erl index bc47772e..e30b64ed 100644 --- a/inttest/tdeps3/tdeps3_rt.erl +++ b/inttest/tdeps3/tdeps3_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(tdeps3_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/tdeps_prefer/tdeps_prefer_rt.erl b/inttest/tdeps_prefer/tdeps_prefer_rt.erl index 94c9b8fd..aca0b96b 100644 --- a/inttest/tdeps_prefer/tdeps_prefer_rt.erl +++ b/inttest/tdeps_prefer/tdeps_prefer_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(tdeps_prefer_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/tdeps_update/tdeps_update_rt.erl b/inttest/tdeps_update/tdeps_update_rt.erl index b4919c50..2710534b 100644 --- a/inttest/tdeps_update/tdeps_update_rt.erl +++ b/inttest/tdeps_update/tdeps_update_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(tdeps_update_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/thooks/fish.erl b/inttest/thooks/fish.erl index 739cb94b..f22194ac 100644 --- a/inttest/thooks/fish.erl +++ b/inttest/thooks/fish.erl @@ -1,5 +1,5 @@ -module(fish). --compile(export_all). +-export([fish/0]). fish() -> fish. diff --git a/inttest/thooks/thooks_rt.erl b/inttest/thooks/thooks_rt.erl index f1e7fa48..55013233 100644 --- a/inttest/thooks/thooks_rt.erl +++ b/inttest/thooks/thooks_rt.erl @@ -3,7 +3,7 @@ -module(thooks_rt). -include_lib("eunit/include/eunit.hrl"). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), diff --git a/inttest/tplugins/bad_plugin.erl b/inttest/tplugins/bad_plugin.erl index 77ec01bb..5730a6d1 100644 --- a/inttest/tplugins/bad_plugin.erl +++ b/inttest/tplugins/bad_plugin.erl @@ -1,5 +1,6 @@ -module(bad_plugin). --compile(export_all). + +-export([fwibble/2]). %% this plugin contains numerous DELIBERATE syntax errors diff --git a/inttest/tplugins/fish.erl b/inttest/tplugins/fish.erl index 739cb94b..aa0639d2 100644 --- a/inttest/tplugins/fish.erl +++ b/inttest/tplugins/fish.erl @@ -1,5 +1,5 @@ -module(fish). --compile(export_all). +-expoort([fish/0]). fish() -> fish. diff --git a/inttest/tplugins/test_plugin.erl b/inttest/tplugins/test_plugin.erl index 461247cc..5b8494f1 100644 --- a/inttest/tplugins/test_plugin.erl +++ b/inttest/tplugins/test_plugin.erl @@ -1,5 +1,6 @@ -module(test_plugin). --compile(export_all). + +-export([fwibble/2]). fwibble(Config, _) -> Pwd = rebar_utils:get_cwd(), diff --git a/inttest/tplugins/tplugins_rt.erl b/inttest/tplugins/tplugins_rt.erl index d36afdd9..c833f627 100644 --- a/inttest/tplugins/tplugins_rt.erl +++ b/inttest/tplugins/tplugins_rt.erl @@ -1,7 +1,7 @@ %% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 et -module(tplugins_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). -include_lib("eunit/include/eunit.hrl"). diff --git a/inttest/vsn_cache/vsn_cache_rt.erl b/inttest/vsn_cache/vsn_cache_rt.erl index b4828880..18c98ab6 100644 --- a/inttest/vsn_cache/vsn_cache_rt.erl +++ b/inttest/vsn_cache/vsn_cache_rt.erl @@ -2,7 +2,7 @@ %% ex: ts=4 sw=4 et -module(vsn_cache_rt). --compile(export_all). +-export([setup/1, files/0, run/1]). setup([Target]) -> retest_utils:load_module(filename:join(Target, "inttest_utils.erl")), From 920e314a90865df129009f9503476133c09df905 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sat, 19 Aug 2017 21:08:26 +0000 Subject: [PATCH 79/89] Remove profiler note If and when getopt gains the required feature, this can be revisited, but let's remove the message, since it's been a long time. --- src/rebar.erl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/rebar.erl b/src/rebar.erl index 6f78fe66..d35e492b 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -156,9 +156,6 @@ init_config1(BaseConfig) -> rebar_config:set_xconf(BaseConfig1, base_dir, AbsCwd). profile(BaseConfig1, Commands) -> - ?CONSOLE("Please take note that profiler=[fprof|eflame] is preliminary" - " and will be~nreplaced with a different command line flag" - " in the next release.~n", []), Profiler = rebar_config:get_global(BaseConfig1, profiler, "fprof"), profile(BaseConfig1, Commands, list_to_atom(Profiler)). From 0eb9ba8d771a2369d324427c57ad675cc7c5731a Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sun, 20 Aug 2017 19:14:29 +0000 Subject: [PATCH 80/89] Revert "Drop crypto app reliance" --- ebin/rebar.app | 1 + rebar.config | 3 +-- src/rebar.erl | 6 ++++++ src/rebar_ct.erl | 12 +++--------- src/rebar_eunit.erl | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ebin/rebar.app b/ebin/rebar.app index b4c8b5a1..9449e1e4 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -55,6 +55,7 @@ stdlib, sasl, compiler, + crypto, syntax_tools, tools, eunit, diff --git a/rebar.config b/rebar.config index 6765415c..eda5a2c8 100644 --- a/rebar.config +++ b/rebar.config @@ -32,8 +32,7 @@ - (\"diameter_dict_util\":\"parse\"/\"2\") - (\"erlang\":\"timestamp\"/\"0\") - (\"rebar_rnd\":\"seed\"/\"1\") - - (\"rebar_rnd\":\"uniform\"/\"0\") - - (\"rebar_rnd\":\"uniform\"/\"1\"))", + - (\"rebar_rnd\":\"uniform\"/\"0\"))", []}]}. {dialyzer, diff --git a/src/rebar.erl b/src/rebar.erl index c87e5912..d35e492b 100644 --- a/src/rebar.erl +++ b/src/rebar.erl @@ -205,6 +205,12 @@ profile(_Config, _Commands, Profiler) -> ?ABORT("Unsupported profiler: ~s~n", [Profiler]). run_aux(BaseConfig, Commands) -> + %% Make sure crypto is running + case crypto:start() of + ok -> ok; + {error,{already_started,crypto}} -> ok + end, + %% Make sure memoization server is running case rmemo:start() of {ok, _} -> ok; diff --git a/src/rebar_ct.erl b/src/rebar_ct.erl index 892ae036..b27f6617 100644 --- a/src/rebar_ct.erl +++ b/src/rebar_ct.erl @@ -237,7 +237,6 @@ make_cmd(TestDir, RawLogDir, Config) -> CodeDirs = [io_lib:format("\"~s\"", [Dir]) || Dir <- [EbinDir|NonLibCodeDirs]], CodePathString = string:join(CodeDirs, " "), - _ = rebar_rnd:seed({55, seconds(), 7331}), Cmd = case get_ct_specs(Config, search_ct_specs_from(Cwd, TestDir, Config)) of undefined -> ?FMT("~s" @@ -286,18 +285,13 @@ search_ct_specs_from(Cwd, TestDir, Config) -> Cwd end. -seconds() -> - calendar:datetime_to_gregorian_seconds(calendar:universal_time()). - build_name(Config) -> %% generate a unique name for our test node, we want %% to make sure the odds of name clashing are low - Secs = integer_to_list(seconds()), - Random = integer_to_list(rebar_rnd:uniform(1000000)), - PseudoUnique = Random ++ "N" ++ Secs, + Random = integer_to_list(crypto:rand_uniform(0, 10000)), case rebar_config:get_local(Config, ct_use_short_names, false) of - true -> "-sname test" ++ PseudoUnique; - false -> " -name test" ++ PseudoUnique ++ "@" ++ net_adm:localhost() + true -> "-sname test" ++ Random; + false -> " -name test" ++ Random ++ "@" ++ net_adm:localhost() end. get_extra_params(Config) -> diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index 538f88ba..ebf76bcf 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -279,7 +279,7 @@ randomize_suites(Config, Modules) -> undefined -> Modules; "true" -> - Seed = rebar_rnd:uniform(65535), + Seed = crypto:rand_uniform(1, 65535), randomize_suites1(Modules, Seed); String -> try list_to_integer(String) of From b322703394d5c0fe9dd4b89d6c20c761b32169a0 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sat, 26 Aug 2017 11:48:46 +0000 Subject: [PATCH 81/89] travis-ci: otp 20.0.2 -> 20.0.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7af52589..dcb66065 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ otp_release: - 17.0 - 18.0 - 19.1 - - 20.0.2 + - 20.0.4 before_script: - hostname -f - cc -v From 2276f85150d6274961fd007ce688f71591a47ada Mon Sep 17 00:00:00 2001 From: Julius Andrikonis Date: Mon, 29 Jan 2018 11:15:02 +0200 Subject: [PATCH 82/89] Log out success message with newlines The raw log file is split into lines to check for errors. However the logged message should contain newlines for easier reading. --- THANKS | 1 + src/rebar_ct.erl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index ab7935c6..943ad94d 100644 --- a/THANKS +++ b/THANKS @@ -150,3 +150,4 @@ John Daily Yury Gargay Frank Hunleth Matwey Kornilov +Julius Andrikonis diff --git a/src/rebar_ct.erl b/src/rebar_ct.erl index b27f6617..c033c58e 100644 --- a/src/rebar_ct.erl +++ b/src/rebar_ct.erl @@ -193,7 +193,7 @@ check_log(Config,RawLogFilename,Fun) -> ?FAIL; true -> - Fun(Msg) + Fun(string:join(Msg, "\n")) end. From b36e72bc8ad0c6e654cdf627304b41911e923b26 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Tue, 30 Aug 2016 16:18:04 +0200 Subject: [PATCH 83/89] Run Dialyzer on Travis-CI --- .travis.yml | 16 +++++++++++++++- Makefile | 12 +++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index dcb66065..7f87c155 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,22 @@ otp_release: - 18.0 - 19.1 - 20.0.4 +env: MAKE_TARGET=ci before_script: - hostname -f - cc -v - ld -v -script: "make travis" + - find .rebar +after_script: + - find .rebar +matrix: + include: + - otp_release: 20.0.4 + env: MAKE_TARGET=ci-dialyze +script: make $MAKE_TARGET +cache: + directories: + - .rebar +before_cache: + - rm -fv .rebar/erlcinfo + - rm -fv $HOME/.cache/.rebar/erlcinfo diff --git a/Makefile b/Makefile index d0d16f88..106e289e 100644 --- a/Makefile +++ b/Makefile @@ -22,11 +22,11 @@ check: debug xref dialyzer deps test xref: @./rebar xref -build_plt: - @./rebar build-plt +maybe_build_plt: + @./rebar -vv check-plt || ./rebar -vv build-plt -dialyzer: - @./rebar dialyze +dialyze: maybe_build_plt + @./rebar -vv dialyze binary: VSN = $(shell ./rebar -V) binary: clean all @@ -45,4 +45,6 @@ test_eunit: all test_inttest: all deps @$(RETEST) -l $(LOG_LEVEL) $(RT_TARGETS) -travis: clean debug xref clean all deps test +ci: clean debug xref clean all deps test + +ci-dialyze: clean debug dialyze From 903c89dc9c5b0c7171003f11b8ac134db9736f3a Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sun, 20 Aug 2017 18:48:08 +0000 Subject: [PATCH 84/89] rebar_utils: fix Dialyzer warning --- src/rebar_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index a5cc0ffd..6a62d34f 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -296,7 +296,7 @@ vcs_vsn(Config, Vsn, Dir) -> VsnString = vcs_vsn_1(Vsn, Dir), Cache1 = dict:store(Key, VsnString, Cache), Config1 = rebar_config:set_xconf(Config, vsn_cache, Cache1), - save_vsn_cache(Config1), + _ = save_vsn_cache(Config1), {Config1, VsnString}; {ok, VsnString} -> {Config, VsnString} From b05882aa97168c2abf55257b358227eaedcc1c8b Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sun, 20 Aug 2017 18:54:31 +0000 Subject: [PATCH 85/89] rebar_cover: ignore opaque type match Dialyzer warning --- src/rebar_cover_utils.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rebar_cover_utils.erl b/src/rebar_cover_utils.erl index 3fcba631..7b85f1cf 100644 --- a/src/rebar_cover_utils.erl +++ b/src/rebar_cover_utils.erl @@ -35,6 +35,8 @@ -include("rebar.hrl"). +-dialyzer({no_opaque, has_header/2}). + %% ==================================================================== %% Internal functions %% ==================================================================== From 73feb5f761f8f37525299cdb1019fda0381ff6c0 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Sun, 20 Aug 2017 18:58:29 +0000 Subject: [PATCH 86/89] rebar_xref: ignore opaque type match Dialyzer warning --- src/rebar_xref.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rebar_xref.erl b/src/rebar_xref.erl index ee3414dc..ddfb0298 100644 --- a/src/rebar_xref.erl +++ b/src/rebar_xref.erl @@ -40,6 +40,8 @@ %% for internal use only -export([info/2]). +-dialyzer({no_opaque, find_function_source/4}). + %% =================================================================== %% Public API %% =================================================================== From 4e372a3558ea5ebc72d677039b0a843a6b64aff7 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Wed, 27 Sep 2017 18:45:55 +0000 Subject: [PATCH 87/89] travis-ci: enable 20.1 --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7f87c155..74440e61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,8 @@ otp_release: - 17.0 - 18.0 - 19.1 - - 20.0.4 + - 20.0.5 + - 20.1 env: MAKE_TARGET=ci before_script: - hostname -f @@ -22,7 +23,7 @@ after_script: - find .rebar matrix: include: - - otp_release: 20.0.4 + - otp_release: 20.1 env: MAKE_TARGET=ci-dialyze script: make $MAKE_TARGET cache: From 16d5dfed70e842d32f46728743aea39c8064b6e5 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Tue, 14 Nov 2017 20:56:48 +0000 Subject: [PATCH 88/89] travis-ci: update otp versions --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74440e61..edd413fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ otp_release: - 18.0 - 19.1 - 20.0.5 - - 20.1 + - 20.1.5 env: MAKE_TARGET=ci before_script: - hostname -f @@ -23,7 +23,7 @@ after_script: - find .rebar matrix: include: - - otp_release: 20.1 + - otp_release: 20.1.5 env: MAKE_TARGET=ci-dialyze script: make $MAKE_TARGET cache: From 8a2aca081780338330cd2145ab6cb86acde7c783 Mon Sep 17 00:00:00 2001 From: Tuncer Ayaz Date: Tue, 30 Jan 2018 23:40:57 +0000 Subject: [PATCH 89/89] travis-ci: allow Dialyzer job to fail --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index edd413fb..8be68140 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,9 @@ matrix: include: - otp_release: 20.1.5 env: MAKE_TARGET=ci-dialyze + allow_failures: + - otp_release: 20.1.5 + env: MAKE_TARGET=ci-dialyze script: make $MAKE_TARGET cache: directories: