From b41f0f1528b390cb7d2f8233453942fb17e1558f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 25 Apr 2025 19:22:22 +0200 Subject: [PATCH 1/3] Load modules lazily --- lib/elixir/lib/kernel/parallel_compiler.ex | 56 ++++++++++++++++++---- lib/elixir/lib/module.ex | 8 ++-- lib/elixir/src/elixir_erl_compiler.erl | 2 + lib/elixir/src/elixir_module.erl | 18 +++---- lib/mix/lib/mix/compilers/elixir.ex | 3 -- 5 files changed, 65 insertions(+), 22 deletions(-) diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index ec64c107d50..053c49550c7 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -181,7 +181,7 @@ defmodule Kernel.ParallelCompiler do {:ok, [atom], [warning] | info()} | {:error, [error] | [Code.diagnostic(:error)], [warning] | info()} def compile_to_path(files, path, options \\ []) when is_binary(path) and is_list(options) do - spawn_workers(files, {:compile, path}, options) + spawn_workers(files, {:compile, path}, Keyword.put(options, :dest, path)) end @doc """ @@ -320,6 +320,9 @@ defmodule Kernel.ParallelCompiler do end defp write_module_binaries(result, {:compile, path}, timestamp) do + File.mkdir_p!(path) + Code.prepend_path(path) + Enum.flat_map(result, fn {{:module, module}, binary} when is_binary(binary) -> full_path = Path.join(path, Atom.to_string(module) <> ".beam") @@ -420,8 +423,8 @@ defmodule Kernel.ParallelCompiler do try do case output do - {:compile, path} -> compile_file(file, path, parent) - :compile -> compile_file(file, dest, parent) + {:compile, _} -> compile_file(file, dest, false, parent) + :compile -> compile_file(file, dest, true, parent) :require -> require_file(file, parent) end catch @@ -527,9 +530,9 @@ defmodule Kernel.ParallelCompiler do wait_for_messages([], spawned, waiting, files, result, warnings, errors, state) end - defp compile_file(file, path, parent) do + defp compile_file(file, path, force_load?, parent) do :erlang.process_flag(:error_handler, Kernel.ErrorHandler) - :erlang.put(:elixir_compiler_dest, path) + :erlang.put(:elixir_compiler_dest, {path, force_load?}) :elixir_compiler.file(file, &each_file(&1, &2, parent)) end @@ -630,19 +633,30 @@ defmodule Kernel.ParallelCompiler do state ) - {:module_available, child, ref, file, module, binary} -> + {:module_available, child, ref, file, module, binary, loaded?} -> state.each_module.(file, module, binary) + available = + case Map.get(result, {:module, module}) do + [_ | _] = pids -> + # We prefer to load in the client, if possible, + # to avoid locking the compilation server. + loaded? or load_module(module, binary, state) + Enum.map(pids, &{&1, :found}) + + _ -> + [] + end + # Release the module loader which is waiting for an ack send(child, {ref, :ack}) - {available, result} = update_result(result, :module, module, binary) spawn_workers( available ++ queue, spawned, waiting, files, - result, + Map.put(result, {:module, module}, binary), warnings, errors, state @@ -661,6 +675,8 @@ defmodule Kernel.ParallelCompiler do {waiting, files, result} = if not is_list(available_or_pending) or on in defining do + # If what we are waiting on was defined but not loaded, we do it now. + load_pending(kind, on, result, state) send(child_pid, {ref, :found}) {waiting, files, result} else @@ -755,6 +771,30 @@ defmodule Kernel.ParallelCompiler do {{:error, Enum.reverse(errors, fun.()), info}, state} end + defp load_pending(kind, module, result, state) do + with true <- kind in [:module, :struct], + %{{:module, ^module} => binary} when is_binary(binary) <- result, + false <- :erlang.module_loaded(module) do + load_module(module, binary, state) + end + end + + defp load_module(module, binary, state) do + beam_location = + case state.dest do + nil -> + [] + + dest -> + :filename.join( + :elixir_utils.characters_to_list(dest), + Atom.to_charlist(module) ++ ~c".beam" + ) + end + + :code.load_binary(module, beam_location, binary) + end + defp update_result(result, kind, module, value) do available = case Map.get(result, {kind, module}) do diff --git a/lib/elixir/lib/module.ex b/lib/elixir/lib/module.ex index 8c388d9b487..7eeb4f1bcb6 100644 --- a/lib/elixir/lib/module.ex +++ b/lib/elixir/lib/module.ex @@ -586,9 +586,11 @@ defmodule Module do name/arity pairs. Inlining is applied locally, calls from another module are not affected by this option - * `@compile {:autoload, false}` - disables automatic loading of - modules after compilation. Instead, the module will be loaded after - it is dispatched to + * `@compile {:autoload, true}` - configures if modules are automatically + loaded after definition. It defaults to `false` when compiling modules + to `.beam` files in disk (as the modules are then lazily loaded from + disk). If modules are not compiled to disk, then they are always loaded, + regardless of this flag * `@compile {:no_warn_undefined, Mod}` or `@compile {:no_warn_undefined, {Mod, fun, arity}}` - does not warn if diff --git a/lib/elixir/src/elixir_erl_compiler.erl b/lib/elixir/src/elixir_erl_compiler.erl index 90848fb3b48..7650f9dcb24 100644 --- a/lib/elixir/src/elixir_erl_compiler.erl +++ b/lib/elixir/src/elixir_erl_compiler.erl @@ -4,6 +4,7 @@ spawn(Fun) -> CompilerInfo = get(elixir_compiler_info), + {error_handler, ErrorHandler} = erlang:process_info(self(), error_handler), CodeDiagnostics = case get(elixir_code_diagnostics) of @@ -13,6 +14,7 @@ spawn(Fun) -> {_, Ref} = spawn_monitor(fun() -> + erlang:process_flag(error_handler, ErrorHandler), put(elixir_compiler_info, CompilerInfo), put(elixir_code_diagnostics, CodeDiagnostics), diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index 66455a23cb4..d34e2c88c83 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -155,7 +155,7 @@ compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) -> put_compiler_modules([Module | CompilerModules]), {Result, ModuleE, CallbackE} = eval_form(Line, Module, DataBag, Block, Vars, Prune, E), CheckerInfo = checker_info(), - BeamLocation = beam_location(ModuleAsCharlist), + {BeamLocation, Forceload} = beam_location(ModuleAsCharlist), {Binary, PersistedAttributes, Autoload} = elixir_erl_compiler:spawn(fun() -> @@ -215,17 +215,17 @@ compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) -> compile_error_if_tainted(DataSet, E), Binary = elixir_erl:compile(ModuleMap), - Autoload = proplists:get_value(autoload, CompileOpts, true), + Autoload = Forceload or proplists:get_value(autoload, CompileOpts, false), spawn_parallel_checker(CheckerInfo, Module, ModuleMap), {Binary, PersistedAttributes, Autoload} end), Autoload andalso code:load_binary(Module, BeamLocation, Binary), + make_module_available(Module, Binary, Autoload), put_compiler_modules(CompilerModules), eval_callbacks(Line, DataBag, after_compile, [CallbackE, Binary], CallbackE), elixir_env:trace({on_module, Binary, none}, ModuleE), warn_unused_attributes(DataSet, DataBag, PersistedAttributes, E), - make_module_available(Module, Binary), (element(2, CheckerInfo) == nil) andalso [VerifyMod:VerifyFun(Module) || {VerifyMod, VerifyFun} <- bag_lookup_element(DataBag, {accumulate, after_verify}, 2)], @@ -544,10 +544,12 @@ bag_lookup_element(Table, Name, Pos) -> beam_location(ModuleAsCharlist) -> case get(elixir_compiler_dest) of - Dest when is_binary(Dest) -> - filename:join(elixir_utils:characters_to_list(Dest), ModuleAsCharlist ++ ".beam"); + {Dest, ForceLoad} when is_binary(Dest) -> + BeamLocation = + filename:join(elixir_utils:characters_to_list(Dest), ModuleAsCharlist ++ ".beam"), + {BeamLocation, ForceLoad}; _ -> - "" + {"", true} end. %% Integration with elixir_compiler that makes the module available @@ -568,7 +570,7 @@ spawn_parallel_checker(CheckerInfo, Module, ModuleMap) -> end, 'Elixir.Module.ParallelChecker':spawn(CheckerInfo, Module, ModuleMap, Log). -make_module_available(Module, Binary) -> +make_module_available(Module, Binary, Loaded) -> case get(elixir_module_binaries) of Current when is_list(Current) -> put(elixir_module_binaries, [{Module, Binary} | Current]); @@ -581,7 +583,7 @@ make_module_available(Module, Binary) -> ok; {PID, _} -> Ref = make_ref(), - PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary}, + PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary, Loaded}, receive {Ref, ack} -> ok end end. diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex index 629bc73cd04..5df2c1286e1 100644 --- a/lib/mix/lib/mix/compilers/elixir.ex +++ b/lib/mix/lib/mix/compilers/elixir.ex @@ -166,9 +166,6 @@ defmodule Mix.Compilers.Elixir do Mix.Utils.compiling_n(length(stale), :ex) Mix.Project.ensure_structure() - - # We don't want to cache this path as we will write to it - true = Code.prepend_path(dest) previous_opts = set_compiler_opts(opts) try do From d0fb3570310ec386b881accb7935ed8a63336ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 26 Apr 2025 09:01:07 +0200 Subject: [PATCH 2/3] Avoid purging non-loaded modules --- lib/mix/lib/mix/compilers/elixir.ex | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex index 5df2c1286e1..6dd57e45c1b 100644 --- a/lib/mix/lib/mix/compilers/elixir.ex +++ b/lib/mix/lib/mix/compilers/elixir.ex @@ -735,16 +735,22 @@ defmodule Mix.Compilers.Elixir do defp remove_and_purge(beam, module) do _ = File.rm(beam) - :code.purge(module) - :code.delete(module) + + if Code.loaded?(module) do + :code.purge(module) + :code.delete(module) + end end defp purge_modules_in_path(path) do with {:ok, beams} <- File.ls(path) do Enum.each(beams, fn beam -> module = beam |> Path.rootname() |> String.to_atom() - :code.purge(module) - :code.delete(module) + + if Code.loaded?(module) do + :code.purge(module) + :code.delete(module) + end end) end end @@ -922,9 +928,7 @@ defmodule Mix.Compilers.Elixir do end for {module, _} <- data do - File.rm(beam_path(compile_path, module)) - :code.purge(module) - :code.delete(module) + remove_and_purge(beam_path(compile_path, module), module) end rescue _ -> From 1cb8e99366f773373d6cade6e50de913da4e6161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 26 Apr 2025 16:57:31 +0200 Subject: [PATCH 3/3] Perform more loading on the client if possible --- lib/elixir/lib/kernel/parallel_compiler.ex | 5 +++++ lib/elixir/src/elixir_module.erl | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index 053c49550c7..3d2d0cee89f 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -633,6 +633,11 @@ defmodule Kernel.ParallelCompiler do state ) + {:module_pending, child, ref, module} -> + pending? = match?(%{{:module, ^module} => [_ | _]}, result) + send(child, {ref, pending?}) + spawn_workers(queue, spawned, waiting, files, result, warnings, errors, state) + {:module_available, child, ref, file, module, binary, loaded?} -> state.each_module.(file, module, binary) diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index d34e2c88c83..e6847670d0a 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -215,7 +215,8 @@ compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) -> compile_error_if_tainted(DataSet, E), Binary = elixir_erl:compile(ModuleMap), - Autoload = Forceload or proplists:get_value(autoload, CompileOpts, false), + Autoload = Forceload or proplists:get_value(autoload, CompileOpts, false) or + waiting_for_module(Module), spawn_parallel_checker(CheckerInfo, Module, ModuleMap), {Binary, PersistedAttributes, Autoload} end), @@ -587,6 +588,16 @@ make_module_available(Module, Binary, Loaded) -> receive {Ref, ack} -> ok end end. + waiting_for_module(Module) -> + case get(elixir_compiler_info) of + undefined -> + false; + {PID, _} -> + Ref = make_ref(), + PID ! {module_pending, self(), Ref, Module}, + receive {Ref, Boolean} -> Boolean end + end. + %% Error handling and helpers. %% We've reached the elixir_module or eval internals, skip it with the rest