New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce new "Dbgi" chunk #1367

Merged
merged 2 commits into from Apr 26, 2017
File filter...
Filter file types
Jump to file or symbol
Failed to load files and symbols.
+476 −364
Diff settings

Always

Just for now

@@ -28,12 +28,12 @@
main([BeamFile,AbstrFile]) ->
{ok,_,Chunks0} = beam_lib:all_chunks(BeamFile),
{ok,Abstr} = file:consult(AbstrFile),
Chunks1 = lists:keyreplace("Abst", 1, Chunks0,
{"Abst",term_to_binary({raw_abstract_v1,Abstr})}),
{"CInf",CInf0} = lists:keyfind("CInf", 1, Chunks1),
CInf = fix_options(CInf0),
Chunks = lists:keyreplace("CInf", 1, Chunks1, {"CInf",CInf}),
{ok,Module} = beam_lib:build_module(Chunks),
{"CInf",CInf0} = lists:keyfind("CInf", 1, Chunks0),
{CInf, COpts} = fix_options(CInf0),
Chunks1 = lists:keyreplace("CInf", 1, Chunks0, {"CInf",CInf}),
Chunks2 = lists:keyreplace("Dbgi", 1, Chunks1,
{"Dbgi",term_to_binary({debug_info_v1,erl_abstract_code,{Abstr, COpts}})}),
{ok,Module} = beam_lib:build_module(Chunks2),
ok = file:write_file(BeamFile, Module),
init:stop().

@@ -42,4 +42,4 @@ fix_options(CInf0) ->
{options,Opts0} = lists:keyfind(options, 1, CInf1),
Opts = Opts0 -- [from_asm],
CInf = lists:keyreplace(options, 1, CInf1, {options,Opts}),
term_to_binary(CInf).
{term_to_binary(CInf), Opts}.
@@ -132,12 +132,10 @@
<tag><c>debug_info</c></tag>
<item>
<marker id="debug_info"></marker>
<p>Includes debug information in the form of abstract code
(see
<seealso marker="erts:absform">The Abstract Format</seealso>
in ERTS User's Guide) in the compiled beam module. Tools
such as Debugger, Xref, and Cover require
the debug information to be included.</p>
<p>Includes debug information in the form of <seealso marker="erts:absform">
Erlang Abstract Format</seealso> in the <c>debug_info</c>
chunk of the compiled beam module. Tools such as Debugger,
Xref, and Cover require the debug information to be included.</p>

<p><em>Warning</em>: Source code can be reconstructed from
the debug information. Use encrypted debug information
@@ -147,6 +145,21 @@
<seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p>
</item>

<tag><c>{debug_info, {Backend, Data}}</c></tag>
<item>
<marker id="debug_info"></marker>
<p>Includes custom debug information in the form of a
<c>Backend</c> module with custom <c>Data</c> in the compiled beam module.
The given module must implement a <c>debug_info/4</c> function
and is responsible for generating different code representations,
as described in the <c>debug_info</c> under
<seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p>

<p><em>Warning</em>: Source code can be reconstructed from
the debug information. Use encrypted debug information
(<c>encrypt_debug_info</c>) to prevent this.</p>
</item>

<tag><c>{debug_info_key,KeyString}</c></tag>
<item></item>
<tag><c>{debug_info_key,{Mode,KeyString}}</c></tag>
Copy path View file
@@ -198,9 +198,9 @@ expand_opts(Opts0) ->
%% {debug_info_key,Key} implies debug_info.
Opts = case {proplists:get_value(debug_info_key, Opts0),
proplists:get_value(encrypt_debug_info, Opts0),
proplists:get_bool(debug_info, Opts0)} of
proplists:get_value(debug_info, Opts0)} of
{undefined,undefined,_} -> Opts0;
{_,_,false} -> [debug_info|Opts0];
{_,_,undefined} -> [debug_info|Opts0];
{_,_,_} -> Opts0
end,
foldr(fun expand_opt/2, [], Opts).
@@ -302,7 +302,7 @@ format_error_reason(Reason) ->
ofile="" :: file:filename(),
module=[] :: module() | [],
core_code=[] :: cerl:c_module() | [],
abstract_code=[] :: binary() | [], %Abstract code for debugger.
abstract_code=[] :: abstract_code(), %Abstract code for debugger.
options=[] :: [option()], %Options for compilation
mod_options=[] :: [option()], %Options for module_info
encoding=none :: none | epp:source_encoding(),
@@ -1315,51 +1315,62 @@ core_inline_module(Code0, #compile{options=Opts}=St) ->
Code = cerl_inline:core_transform(Code0, Opts),
{ok,Code,St}.

save_abstract_code(Code, #compile{ifile=File}=St) ->
case abstract_code(Code, St) of
{ok,Abstr} ->
{ok,Code,St#compile{abstract_code=Abstr}};
{error,Es} ->
{error,St#compile{errors=St#compile.errors ++ [{File,Es}]}}
end.
save_abstract_code(Code, St) ->
{ok,Code,St#compile{abstract_code=erl_parse:anno_to_term(Code)}}.

debug_info(#compile{module=Module,mod_options=Opts0,ofile=OFile,abstract_code=Abst}) ->
AbstOpts = cleanup_compile_options(Opts0),
Opts1 = proplists:delete(debug_info, Opts0),
{Backend,Metadata,Opts2} =
case proplists:get_value(debug_info, Opts0, false) of
{OptBackend,OptMetadata} when is_atom(OptBackend) -> {OptBackend,OptMetadata,Opts1};
false -> {erl_abstract_code,{none,AbstOpts},Opts1};
true -> {erl_abstract_code,{Abst,AbstOpts},[debug_info | Opts1]}
end,
DebugInfo = erlang:term_to_binary({debug_info_v1,Backend,Metadata}, [compressed]),

abstract_code(Code0, #compile{options=Opts,ofile=OFile}) ->
Code = erl_parse:anno_to_term(Code0),
Abstr = erlang:term_to_binary({raw_abstract_v1,Code}, [compressed]),
case member(encrypt_debug_info, Opts) of
case member(encrypt_debug_info, Opts2) of
true ->
case keyfind(debug_info_key, 1, Opts) of
{_,Key} ->
encrypt_abs_code(Abstr, Key);
case lists:keytake(debug_info_key, 1, Opts2) of
{value,{_, Key},Opts3} ->
encrypt_debug_info(DebugInfo, Key, [{debug_info_key,'********'} | Opts3]);
false ->
%% Note: #compile.module has not been set yet.
%% Here is an approximation that should work for
%% all valid cases.
Module = list_to_atom(filename:rootname(filename:basename(OFile))),
Mode = proplists:get_value(crypto_mode, Opts, des3_cbc),
Mode = proplists:get_value(crypto_mode, Opts2, des3_cbc),
case beam_lib:get_crypto_key({debug_info, Mode, Module, OFile}) of
error ->
{error, [{none,?MODULE,no_crypto_key}]};
Key ->
encrypt_abs_code(Abstr, {Mode, Key})
encrypt_debug_info(DebugInfo, {Mode, Key}, Opts2)
end
end;
false ->
{ok,Abstr}
{ok,DebugInfo,Opts2}
end.

encrypt_abs_code(Abstr, Key0) ->
encrypt_debug_info(DebugInfo, Key, Opts) ->
try
RealKey = generate_key(Key0),
RealKey = generate_key(Key),
case start_crypto() of
ok -> {ok,encrypt(RealKey, Abstr)};
ok -> {ok,encrypt(RealKey, DebugInfo),Opts};
{error,_}=E -> E
end
catch
error:_ ->
{error,[{none,?MODULE,bad_crypto_key}]}
end.

cleanup_compile_options(Opts) ->
lists:filter(fun keep_compile_option/1, Opts).

%% We are storing abstract, not asm or core.
keep_compile_option(from_asm) -> false;
keep_compile_option(from_core) -> false;
%% Parse transform and macros have already been applied.
keep_compile_option({parse_transform, _}) -> false;
keep_compile_option({d, _, _}) -> false;
%% Do not affect compilation result on future calls.
keep_compile_option(Option) -> effects_code_generation(Option).

start_crypto() ->
try crypto:start() of
{error,{already_started,crypto}} -> ok;
@@ -1386,16 +1397,16 @@ encrypt({des3_cbc=Type,Key,IVec,BlockSize}, Bin0) ->
save_core_code(Code, St) ->
{ok,Code,St#compile{core_code=cerl:from_records(Code)}}.

beam_asm(Code0, #compile{ifile=File,abstract_code=Abst,extra_chunks=ExtraChunks,
options=CompilerOpts,mod_options=Opts0}=St) ->
Source = paranoid_absname(File),
Opts1 = lists:map(fun({debug_info_key,_}) -> {debug_info_key,'********'};
(Other) -> Other
end, Opts0),
Opts2 = [O || O <- Opts1, effects_code_generation(O)],
Chunks = [{<<"Abst">>, Abst} | ExtraChunks],
case beam_asm:module(Code0, Chunks, Source, Opts2, CompilerOpts) of
{ok,Code} -> {ok,Code,St#compile{abstract_code=[]}}
beam_asm(Code0, #compile{ifile=File,extra_chunks=ExtraChunks,options=CompilerOpts}=St) ->
case debug_info(St) of
{ok,DebugInfo,Opts0} ->
Source = paranoid_absname(File),
Opts1 = [O || O <- Opts0, effects_code_generation(O)],
Chunks = [{<<"Dbgi">>, DebugInfo} | ExtraChunks],
{ok,Code} = beam_asm:module(Code0, Chunks, Source, Opts1, CompilerOpts),
{ok,Code,St#compile{abstract_code=[]}};
{error,Es} ->
{error,St#compile{errors=St#compile.errors ++ [{File,Es}]}}
end.

paranoid_absname(""=File) ->
@@ -1479,15 +1490,17 @@ embed_native_code(Code, {Architecture,NativeCode}) ->
%% errors will be reported).

effects_code_generation(Option) ->
case Option of
case Option of
beam -> false;
report_warnings -> false;
report_errors -> false;
return_errors-> false;
return_warnings-> false;
warnings_as_errors -> false;
binary -> false;
verbose -> false;
{cwd,_} -> false;
{outdir, _} -> false;
_ -> true
end.

Copy path View file
@@ -184,11 +184,8 @@ form({function,_,_,_,_}=F0, Module, Opts) ->
form({attribute,_,module,Mod}, Module, _Opts) ->
true = is_atom(Mod),
Module#imodule{name=Mod};
form({attribute,_,file,{File,_Line}}, Module, _Opts) ->
Module#imodule{file=File};
form({attribute,_,compile,_}, Module, _Opts) ->
%% Ignore compilation options.
Module;
form({attribute,_,file,{File,_Line}}=F, #imodule{attrs=As}=Module, _Opts) ->
Module#imodule{file=File, attrs=[attribute(F)|As]};

This comment has been minimized.

@josevalim

josevalim Apr 10, 2017

Contributor

The changes above are so we keep both compile and file attributes in the list of core attributes as those are required if we want dialyzer to work exclusively on core.

form({attribute,_,import,_}, Module, _Opts) ->
%% Ignore. We have no futher use for imports.
Module;
@@ -201,9 +198,8 @@ form(_, Module, _Opts) ->
%% Ignore uninteresting forms such as 'eof'.
Module.

attribute(Attribute) ->
Fun = fun(A) -> [erl_anno:location(A)] end,
{attribute,Line,Name,Val0} = erl_parse:map_anno(Fun, Attribute),
attribute({attribute,A,Name,Val0}) ->
Line = [erl_anno:location(A)],
Val = if
is_list(Val0) -> Val0;
true -> [Val0]
@@ -148,6 +148,8 @@ include_attribute(opaque) -> false;
include_attribute(export_type) -> false;
include_attribute(record) -> false;
include_attribute(optional_callbacks) -> false;
include_attribute(file) -> false;
include_attribute(compile) -> false;
include_attribute(_) -> true.

function({#c_var{name={F,Arity}=FA},Body}, St0) ->
@@ -319,7 +319,7 @@ self_compile_1(Config, Prefix, Opts) ->
%% Compile the compiler. (In this node to get better coverage.)
CompA = make_compiler_dir(Priv, Prefix++"compiler_a"),
VsnA = Version ++ ".0",
compile_compiler(compiler_src(), CompA, VsnA, [clint0,clint|Opts]),
compile_compiler(compiler_src(), CompA, VsnA, Opts),

%% Compile the compiler again using the newly compiled compiler.
%% (In another node because reloading the compiler would disturb cover.)
@@ -27,6 +27,7 @@
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
init_per_group/2,end_per_group/2,
app_test/1,appup_test/1,
debug_info/4, custom_debug_info/1,
file_1/1, forms_2/1, module_mismatch/1, big_file/1, outdir/1,
binary/1, makedep/1, cond_and_ifdef/1, listings/1, listings_big/1,
other_output/1, kernel_listing/1, encrypted_abstr/1,
@@ -51,7 +52,7 @@ all() ->
strict_record, utf8_atoms, extra_chunks,
cover, env, core, core_roundtrip, asm, optimized_guards,
sys_pre_attributes, dialyzer, warnings, pre_load_check,
env_compiler_options].
env_compiler_options, custom_debug_info].

groups() ->
[].
@@ -504,17 +505,23 @@ encrypted_abstr_1(Simple, Target) ->
{ok,simple} = compile:file(Simple,
[debug_info,{debug_info_key,Key},
{outdir,TargetDir}]),
verify_abstract(Target),
verify_abstract(Target, erl_abstract_code),

{ok,simple} = compile:file(Simple,
[{debug_info_key,Key},
{outdir,TargetDir}]),
verify_abstract(Target),
verify_abstract(Target, erl_abstract_code),

{ok,simple} = compile:file(Simple,
[debug_info,{debug_info_key,{des3_cbc,Key}},
{outdir,TargetDir}]),
verify_abstract(Target),
verify_abstract(Target, erl_abstract_code),

{ok,simple} = compile:file(Simple,
[{debug_info,{?MODULE,ok}},
{debug_info_key,Key},
{outdir,TargetDir}]),
verify_abstract(Target, ?MODULE),

{ok,{simple,[{compile_info,CInfo}]}} =
beam_lib:chunks(Target, [compile_info]),
@@ -539,7 +546,7 @@ encrypted_abstr_1(Simple, Target) ->
NewKey = "better use another key here",
write_crypt_file(["[{debug_info,des3_cbc,simple,\"",NewKey,"\"}].\n"]),
{ok,simple} = compile:file(Simple, [encrypt_debug_info,report]),
verify_abstract("simple.beam"),
verify_abstract("simple.beam", erl_abstract_code),
ok = file:delete(".erlang.crypt"),
beam_lib:clear_crypto_key_fun(),
{error,beam_lib,{key_missing_or_invalid,"simple.beam",abstract_code}} =
@@ -572,9 +579,10 @@ encrypted_abstr_no_crypto(Simple, Target) ->
{outdir,TargetDir},report]),
ok.

verify_abstract(Target) ->
{ok,{simple,[Chunk]}} = beam_lib:chunks(Target, [abstract_code]),
{abstract_code,{raw_abstract_v1,_}} = Chunk.
verify_abstract(Beam, Backend) ->
{ok,{simple,[Abst, Dbgi]}} = beam_lib:chunks(Beam, [abstract_code, debug_info]),
{abstract_code,{raw_abstract_v1,_}} = Abst,
{debug_info,{debug_info_v1,Backend,_}} = Dbgi.

has_crypto() ->
try
@@ -593,6 +601,26 @@ install_crypto_key(Key) ->
ok = beam_lib:crypto_key_fun(F).

%% Miscellanous tests, mainly to get better coverage.
debug_info(erlang_v1, Module, ok, _Opts) ->
{ok, [Module]};
debug_info(erlang_v1, Module, error, _Opts) ->
{error, unknown_format}.

custom_debug_info(Config) when is_list(Config) ->
{Simple,_} = get_files(Config, simple, "file_1"),

{ok,simple,OkBin} = compile:file(Simple, [binary, {debug_info,{?MODULE,ok}}]), %Coverage
{ok,{simple,[{abstract_code,{raw_abstract_v1,[simple]}}]}} =
beam_lib:chunks(OkBin, [abstract_code]),
{ok,{simple,[{debug_info,{debug_info_v1,?MODULE,ok}}]}} =
beam_lib:chunks(OkBin, [debug_info]),

{ok,simple,ErrorBin} = compile:file(Simple, [binary, {debug_info,{?MODULE,error}}]), %Coverage
{ok,{simple,[{abstract_code,no_abstract_code}]}} =
beam_lib:chunks(ErrorBin, [abstract_code]),
{ok,{simple,[{debug_info,{debug_info_v1,?MODULE,error}}]}} =
beam_lib:chunks(ErrorBin, [debug_info]).

cover(Config) when is_list(Config) ->
io:format("~p\n", [compile:options()]),
ok.
Oops, something went wrong.
ProTip! Use n and p to navigate between commits in a pull request.