From 96212bb096516d3592b43165b694c125db03a6a6 Mon Sep 17 00:00:00 2001 From: sabiwara Date: Sat, 19 Oct 2024 19:57:43 +0900 Subject: [PATCH 1/7] Attemptive fix for edge case in normalizer for keyword args - wip --- lib/elixir/lib/code/normalizer.ex | 2 +- .../code_normalizer/quoted_ast_test.exs | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index 96ec5a826db..c4a71b676cd 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -347,7 +347,7 @@ defmodule Code.Normalizer do args = normalize_args(args, %{state | parent_meta: meta}) {form, meta, args} - Keyword.has_key?(meta, :do) or match?([{{:__block__, _, [:do]}, _} | _], last) -> + Keyword.has_key?(meta, :do) -> # def foo do :ok end # def foo, do: :ok normalize_kw_blocks(form, meta, args, state) diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index cc41cf20eba..d02435ba9d4 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -629,6 +629,45 @@ defmodule Code.Normalizer.QuotedASTTest do assert quoted_to_string(quote(do: foo |> [bar: :baz])) == "foo |> [bar: :baz]" end + test "keyword arg edge case: cursor" do + input = "def foo, do: :bar, __cursor__()" + expected = "def foo, [{:do, :bar}, __cursor__()]" + + ast = Code.string_to_quoted!(input, token_metadata: true) + + assert quoted_to_string(ast) == expected + + ast = + Code.string_to_quoted!(input, + token_metadata: true, + literal_encoder: &{:ok, {:__block__, &2, [&1]}} + ) + + assert quoted_to_string(ast) == expected + end + + test "keyword arg edge case: literal encoder" do + input = """ + foo Bar do + :ok + end + """ + + expected = String.trim(input) + + ast = Code.string_to_quoted!(input, token_metadata: true) + + assert quoted_to_string(ast) == expected + + ast = + Code.string_to_quoted!(input, + token_metadata: true, + literal_encoder: &{:ok, {:__block__, &2, [&1]}} + ) + + assert quoted_to_string(ast) == expected + end + test "list in module attribute" do assert quoted_to_string( quote do From 66775f5bea5d1b876f1c6265d5bf48dd89bda4b4 Mon Sep 17 00:00:00 2001 From: sabiwara Date: Sat, 19 Oct 2024 20:50:20 +0900 Subject: [PATCH 2/7] Failing test --- .../test/elixir/code_normalizer/quoted_ast_test.exs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index d02435ba9d4..1f583cda95d 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -633,13 +633,12 @@ defmodule Code.Normalizer.QuotedASTTest do input = "def foo, do: :bar, __cursor__()" expected = "def foo, [{:do, :bar}, __cursor__()]" - ast = Code.string_to_quoted!(input, token_metadata: true) + ast = Code.string_to_quoted!(input) assert quoted_to_string(ast) == expected ast = Code.string_to_quoted!(input, - token_metadata: true, literal_encoder: &{:ok, {:__block__, &2, [&1]}} ) @@ -648,20 +647,19 @@ defmodule Code.Normalizer.QuotedASTTest do test "keyword arg edge case: literal encoder" do input = """ - foo Bar do + foo(Bar) do :ok end """ expected = String.trim(input) - ast = Code.string_to_quoted!(input, token_metadata: true) + ast = Code.string_to_quoted!(input) assert quoted_to_string(ast) == expected ast = Code.string_to_quoted!(input, - token_metadata: true, literal_encoder: &{:ok, {:__block__, &2, [&1]}} ) From 8a3f7b509c3aa7a26f51c059984dbb070cedb719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 22 Oct 2024 15:14:13 +0200 Subject: [PATCH 3/7] Fixes --- lib/elixir/lib/code/normalizer.ex | 11 +++++-- .../code_normalizer/quoted_ast_test.exs | 32 ++++++++++--------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index c4a71b676cd..3b28bca535e 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -352,9 +352,10 @@ defmodule Code.Normalizer do # def foo, do: :ok normalize_kw_blocks(form, meta, args, state) - match?([{:do, _} | _], last) and Keyword.keyword?(last) -> + (match?([{:do, _} | _], last) and Keyword.keyword?(last)) or + (match?([{{:__block__, _, [:do]}, _} | _], last) and block_keyword?(last)) -> # Non normalized kw blocks - line = state.parent_meta[:line] + line = state.parent_meta[:line] || meta[:line] meta = meta ++ [do: [line: line], end: [line: line]] normalize_kw_blocks(form, meta, args, state) @@ -382,6 +383,12 @@ defmodule Code.Normalizer do end end + defp block_keyword?([{{:__block__, _, [key]}, _val} | tail]) when is_atom(key), + do: block_keyword?(tail) + + defp block_keyword?([]), do: true + defp block_keyword?(_), do: false + defp allow_keyword?(:when, 2), do: true defp allow_keyword?(:{}, _), do: false defp allow_keyword?(op, arity), do: not is_atom(op) or not Macro.operator?(op, arity) diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index 1f583cda95d..e1454cae564 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -629,41 +629,43 @@ defmodule Code.Normalizer.QuotedASTTest do assert quoted_to_string(quote(do: foo |> [bar: :baz])) == "foo |> [bar: :baz]" end - test "keyword arg edge case: cursor" do + test "keyword arg with cursor" do input = "def foo, do: :bar, __cursor__()" expected = "def foo, [{:do, :bar}, __cursor__()]" ast = Code.string_to_quoted!(input) + assert quoted_to_string(ast) == expected + encoder = &{:ok, {:__block__, &2, [&1]}} + ast = Code.string_to_quoted!(input, literal_encoder: encoder) assert quoted_to_string(ast) == expected - ast = - Code.string_to_quoted!(input, - literal_encoder: &{:ok, {:__block__, &2, [&1]}} - ) + ast = Code.string_to_quoted!(input, token_metadata: true) + assert quoted_to_string(ast) == expected + ast = Code.string_to_quoted!(input, literal_encoder: encoder, token_metadata: true) assert quoted_to_string(ast) == expected end - test "keyword arg edge case: literal encoder" do + test "keyword arg with tokenizer options" do input = """ foo(Bar) do :ok - end + end\ """ - expected = String.trim(input) - ast = Code.string_to_quoted!(input) + assert quoted_to_string(ast) == input - assert quoted_to_string(ast) == expected + encoder = &{:ok, {:__block__, &2, [&1]}} + ast = Code.string_to_quoted!(input, literal_encoder: encoder) + assert quoted_to_string(ast) == input - ast = - Code.string_to_quoted!(input, - literal_encoder: &{:ok, {:__block__, &2, [&1]}} - ) + ast = Code.string_to_quoted!(input, token_metadata: true) + assert quoted_to_string(ast) == input - assert quoted_to_string(ast) == expected + ast = Code.string_to_quoted!(input, literal_encoder: encoder, token_metadata: true) + assert quoted_to_string(ast) == input end test "list in module attribute" do From 89621e007f10ab7bb4f23b4270061079472ffd74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 22 Oct 2024 15:43:33 +0200 Subject: [PATCH 4/7] Fix --- lib/elixir/lib/code/normalizer.ex | 13 ++++++++----- .../elixir/code_normalizer/quoted_ast_test.exs | 15 +++------------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index 3b28bca535e..efc24a7ed53 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -352,8 +352,7 @@ defmodule Code.Normalizer do # def foo, do: :ok normalize_kw_blocks(form, meta, args, state) - (match?([{:do, _} | _], last) and Keyword.keyword?(last)) or - (match?([{{:__block__, _, [:do]}, _} | _], last) and block_keyword?(last)) -> + match?([{:do, _} | _], last) and Keyword.keyword?(last) -> # Non normalized kw blocks line = state.parent_meta[:line] || meta[:line] meta = meta ++ [do: [line: line], end: [line: line]] @@ -365,9 +364,13 @@ defmodule Code.Normalizer do last_args = case last_arg do - {:__block__, _, [[{{:__block__, key_meta, _}, _} | _]] = last_args} -> - if key_meta[:format] == :keyword do - last_args + {:__block__, _meta, [[{{:__block__, key_meta, _}, _} | _] = keyword]} -> + if key_meta[:format] == :keyword or block_keyword?(keyword) do + [ + Enum.map(keyword, fn {{:__block__, meta, args}, value} -> + {{:__block__, [format: :keyword] ++ meta, args}, value} + end) + ] else [last_arg] end diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index e1454cae564..8b5877f799a 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -647,25 +647,16 @@ defmodule Code.Normalizer.QuotedASTTest do assert quoted_to_string(ast) == expected end - test "keyword arg with tokenizer options" do + test "keyword arg with literal encoder and no metadata" do input = """ foo(Bar) do :ok - end\ + end """ - ast = Code.string_to_quoted!(input) - assert quoted_to_string(ast) == input - encoder = &{:ok, {:__block__, &2, [&1]}} ast = Code.string_to_quoted!(input, literal_encoder: encoder) - assert quoted_to_string(ast) == input - - ast = Code.string_to_quoted!(input, token_metadata: true) - assert quoted_to_string(ast) == input - - ast = Code.string_to_quoted!(input, literal_encoder: encoder, token_metadata: true) - assert quoted_to_string(ast) == input + assert quoted_to_string(ast) == "foo(Bar, do: :ok)" end test "list in module attribute" do From db6274e52e1325ab489d382a290ae25959cf44bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 22 Oct 2024 16:43:23 +0200 Subject: [PATCH 5/7] Update lib/elixir/lib/code/normalizer.ex Co-authored-by: Jean Klingler --- lib/elixir/lib/code/normalizer.ex | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index efc24a7ed53..8b31f021fe6 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -365,13 +365,18 @@ defmodule Code.Normalizer do last_args = case last_arg do {:__block__, _meta, [[{{:__block__, key_meta, _}, _} | _] = keyword]} -> - if key_meta[:format] == :keyword or block_keyword?(keyword) do - [ - Enum.map(keyword, fn {{:__block__, meta, args}, value} -> - {{:__block__, [format: :keyword] ++ meta, args}, value} - end) - ] - else + cond do + key_meta[:format] == :keyword -> + [keyword] + + block_keyword?(keyword) -> + [ + Enum.map(keyword, fn {{:__block__, meta, args}, value} -> + {{:__block__, [format: :keyword] ++ meta, args}, value} + end) + ] + + true -> [last_arg] end From 8995f37f26381da8f00b4cb127894874c59cb639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 22 Oct 2024 16:45:59 +0200 Subject: [PATCH 6/7] Update lib/elixir/lib/code/normalizer.ex --- lib/elixir/lib/code/normalizer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index 8b31f021fe6..61463755972 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -376,7 +376,7 @@ defmodule Code.Normalizer do end) ] - true -> + true -> [last_arg] end From ed77fc5576df3aca7b3afa2d5491d8a0b4d6ed01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 22 Oct 2024 16:46:17 +0200 Subject: [PATCH 7/7] Update lib/elixir/lib/code/normalizer.ex --- lib/elixir/lib/code/normalizer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex index 61463755972..32189dfc972 100644 --- a/lib/elixir/lib/code/normalizer.ex +++ b/lib/elixir/lib/code/normalizer.ex @@ -377,7 +377,7 @@ defmodule Code.Normalizer do ] true -> - [last_arg] + [last_arg] end [] ->