diff --git a/lib/elixir/src/elixir_erl_compiler.erl b/lib/elixir/src/elixir_erl_compiler.erl
index a26f45faee5..140d057bb70 100644
--- a/lib/elixir/src/elixir_erl_compiler.erl
+++ b/lib/elixir/src/elixir_erl_compiler.erl
@@ -135,6 +135,17 @@ custom_format(sys_core_fold, nomatch_guard) ->
   "this check/guard will always yield the same result";
 
 %% Handle literal eval failures
+custom_format(sys_core_fold, {eval_failure, {Mod, Name, Arity}, Error}) ->
+  #{'__struct__' := Struct} = 'Elixir.Exception':normalize(error, Error),
+  {ExMod, ExName, ExArgs} = elixir_rewrite:erl_to_ex(Mod, Name, lists:duplicate(Arity, nil)),
+  Call = 'Elixir.Exception':format_mfa(ExMod, ExName, length(ExArgs)),
+  Trimmed = case Call of
+              <<"Kernel.", Rest/binary>> -> Rest;
+              _ -> Call
+            end,
+  ["the call to ", Trimmed, " will fail with ", elixir_aliases:inspect(Struct)];
+
+%% TODO: remove when we require OTP 24
 custom_format(sys_core_fold, {eval_failure, Error}) ->
   #{'__struct__' := Struct} = 'Elixir.Exception':normalize(error, Error),
   ["this expression will fail with ", elixir_aliases:inspect(Struct)];
diff --git a/lib/elixir/test/elixir/kernel/warning_test.exs b/lib/elixir/test/elixir/kernel/warning_test.exs
index ca4cc89a15b..34cfe94eb95 100644
--- a/lib/elixir/test/elixir/kernel/warning_test.exs
+++ b/lib/elixir/test/elixir/kernel/warning_test.exs
@@ -974,6 +974,15 @@ defmodule Kernel.WarningTest do
     purge(Sample)
   end
 
+  # TODO: Simplify when we require OTP 24
+  if System.otp_release() >= "24" do
+    @argument_error_message "the call to :erlang.atom_to_binary/2"
+    @arithmetic_error_message "the call to +/2"
+  else
+    @argument_error_message "this expression"
+    @arithmetic_error_message "this expression"
+  end
+
   test "eval failure warning" do
     assert capture_err(fn ->
              Code.eval_string("""
@@ -981,7 +990,7 @@ defmodule Kernel.WarningTest do
                def foo, do: Atom.to_string "abc"
              end
              """)
-           end) =~ ~r"this expression will fail with ArgumentError\n.*nofile:2"
+           end) =~ "#{@argument_error_message} will fail with ArgumentError\n  nofile:2"
 
     assert capture_err(fn ->
              Code.eval_string("""
@@ -989,7 +998,7 @@ defmodule Kernel.WarningTest do
                def foo, do: 1 + nil
              end
              """)
-           end) =~ ~r"this expression will fail with ArithmeticError\n.*nofile:2"
+           end) =~ "#{@arithmetic_error_message} will fail with ArithmeticError\n  nofile:2"
   after
     purge([Sample1, Sample2])
   end