From e6dce2a840f358c68824560849c3f67499427a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Wojtasik?= Date: Fri, 4 Feb 2022 17:37:33 +0100 Subject: [PATCH 1/2] Support Elixir 1.13 In version >= 1.13 some expressions previously without a line now have a line propagated from a parent. This line does not always match the location of the expression in the code. So we still need to look at the tokens to correct these locations. This location presence required small changes in the AstSpecifier code because now we can't be sure when the expression has a line then this line is correct. Changes: - Adapt debug.ex to new elixir api - Add specifying a line in nil. - Propagate a generated flag from clauses to conditions. - Consume the beginning token of list or tuple. - Stop predicting the last line of the form from :bin or :cons expressions. --- lib/gradient/ast_specifier.ex | 50 ++++++++++++++++++++++++----------- lib/gradient/debug.ex | 6 ++++- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/lib/gradient/ast_specifier.ex b/lib/gradient/ast_specifier.ex index 8aa4a4f3..863def5d 100644 --- a/lib/gradient/ast_specifier.ex +++ b/lib/gradient/ast_specifier.ex @@ -203,6 +203,8 @@ defmodule Gradient.AstSpecifier do if not :erl_anno.generated(anno) do context_mapper_fold(args, tokens, opts) else + opts = Keyword.put(opts, :generated, :erl_anno.generated(anno)) + {args, []} = context_mapper_fold(args, [], opts) {args, tokens} end @@ -211,6 +213,8 @@ defmodule Gradient.AstSpecifier do {:clause, anno, args, guards, children} |> pass_tokens(tokens) else + arg_opts = Keyword.put(opts, :generated, :erl_anno.generated(anno)) + {args, []} = context_mapper_fold(args, [], arg_opts) {children, tokens} = children |> context_mapper_fold(tokens, opts) {:clause, anno, args, guards, children} @@ -263,7 +267,6 @@ defmodule Gradient.AstSpecifier do def mapper({:cons, anno, value, more} = cons, tokens, opts) do # anno could be 0 {:ok, line, anno, opts, _} = get_line(anno, opts) - tokens = drop_tokens_to_line(tokens, line) case get_list(tokens, opts) do @@ -286,14 +289,16 @@ defmodule Gradient.AstSpecifier do def mapper({:tuple, anno, elements}, tokens, opts) do # anno could be 0 - {:ok, line, anno, opts, has_line?} = get_line(anno, opts) + {:ok, line, anno, opts, _} = get_line(anno, opts) tokens |> drop_tokens_to_line(line) |> get_tuple(opts) |> case do {:tuple, tokens} -> - {anno, opts} = update_line_from_tokens(tokens, anno, opts, has_line?) + {anno, opts} = update_line_from_tokens(tokens, anno, opts) + # drop a token that begins tuple + tokens = drop_tokens_while(tokens, fn t -> elem(t, 0) in [:"{"] end) {elements, tokens} = context_mapper_fold(elements, tokens, opts) @@ -412,12 +417,21 @@ defmodule Gradient.AstSpecifier do end end - def mapper({type, 0, value}, tokens, opts) + def mapper({nil, _}, tokens, opts) do + {:ok, line} = Keyword.fetch(opts, :line) + + {nil, line} + |> pass_tokens(tokens) + end + + def mapper({type, anno, value}, tokens, opts) when type in [:atom, :char, :float, :integer, :string, :bin] do # TODO check what happend for :string {:ok, line} = Keyword.fetch(opts, :line) + anno = :erl_anno.set_line(line, anno) + anno = :erl_anno.set_generated(Keyword.get(opts, :generated, false), anno) - {type, line, value} + {type, anno, value} |> specify_line(tokens, opts) end @@ -560,9 +574,11 @@ defmodule Gradient.AstSpecifier do """ @spec cons_mapper(form(), [token()], options()) :: {form(), tokens()} def cons_mapper({:cons, anno, value, tail}, tokens, opts) do - {:ok, _, anno, opts, has_line?} = get_line(anno, opts) + {:ok, _, anno0, opts0, _} = get_line(anno, opts) - {anno, opts} = update_line_from_tokens(tokens, anno, opts, has_line?) + {anno, opts} = update_line_from_tokens(tokens, anno0, opts0) + # drop a token that begins list + tokens = drop_tokens_while(tokens, fn t -> elem(t, 0) in [:"["] end) {new_value, tokens} = mapper(value, tokens, opts) @@ -720,18 +736,14 @@ defmodule Gradient.AstSpecifier do {:cons, loc, {:integer, loc, value}, charlist_set_loc(tail, loc)} end - def charlist_set_loc({nil, loc}, _), do: {nil, loc} + def charlist_set_loc({nil, _}, loc), do: {nil, loc} - def put_line(anno, opts, line) do - {:erl_anno.set_line(line, anno), Keyword.put(opts, :line, line)} - end - - def update_line_from_tokens([token | _], anno, opts, false) do + def update_line_from_tokens([token | _], anno, opts) do line = get_line_from_token(token) - put_line(anno, opts, line) + {:erl_anno.set_line(line, anno), Keyword.put(opts, :line, line)} end - def update_line_from_tokens(_, anno, opts, _) do + def update_line_from_tokens(_, anno, opts) do {anno, opts} end @@ -761,6 +773,14 @@ defmodule Gradient.AstSpecifier do end defp set_form_end_line(opts, form, forms) do + if elem(form, 0) not in [:bin, :cons] do + set_form_end_line_(opts, form, forms) + else + opts + end + end + + defp set_form_end_line_(opts, form, forms) do case Enum.find(forms, fn f -> anno = elem(f, 1) diff --git a/lib/gradient/debug.ex b/lib/gradient/debug.ex index 6633ad16..77ed6572 100644 --- a/lib/gradient/debug.ex +++ b/lib/gradient/debug.ex @@ -24,7 +24,11 @@ defmodule Gradient.Debug do @spec quoted_to_ast(elixir_form()) :: erlang_form() def quoted_to_ast(qt) do env = :elixir_env.new() - {ast, _, _} = :elixir.quoted_to_erl(qt, env) + + ast = + :elixir.quoted_to_erl(qt, env) + |> elem(0) + Macro.escape(ast) end From 33bd9014b92c54fd1da304828aad9a5d893e2311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Wojtasik?= Date: Fri, 4 Feb 2022 17:40:57 +0100 Subject: [PATCH 2/2] Adapt tests to the changes in AstSpecifier and Elixir 1.13 --- test/gradient/ast_specifier_test.exs | 108 ++++++++++++++------------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/test/gradient/ast_specifier_test.exs b/test/gradient/ast_specifier_test.exs index 633daa7e..0f5a2d89 100644 --- a/test/gradient/ast_specifier_test.exs +++ b/test/gradient/ast_specifier_test.exs @@ -68,7 +68,7 @@ defmodule Gradient.AstSpecifierTest do {:clause, 2, [], [], [ {:cons, 2, {:integer, 2, 97}, - {:cons, 2, {:integer, 2, 98}, {:cons, 2, {:integer, 2, 99}, {nil, 0}}}} + {:cons, 2, {:integer, 2, 98}, {:cons, 2, {:integer, 2, 99}, {nil, 2}}}} ]} ]} = inline @@ -77,7 +77,7 @@ defmodule Gradient.AstSpecifierTest do {:clause, 4, [], [], [ {:cons, 5, {:integer, 5, 97}, - {:cons, 5, {:integer, 5, 98}, {:cons, 5, {:integer, 5, 99}, {nil, 0}}}} + {:cons, 5, {:integer, 5, 98}, {:cons, 5, {:integer, 5, 99}, {nil, 5}}}} ]} ]} = block end @@ -123,7 +123,7 @@ defmodule Gradient.AstSpecifierTest do {:cons, 20, {:tuple, 20, [{:atom, 20, :pretty}, {:atom, 20, true}]}, {:cons, 20, {:tuple, 20, [{:atom, 20, :limit}, {:atom, 20, :infinity}]}, - {nil, 0}}} + {nil, 20}}} ]}, :default, [:binary]}, {:bin_element, 20, {:string, 20, ' using default \n'}, :default, :default} ]}}, @@ -188,7 +188,7 @@ defmodule Gradient.AstSpecifierTest do ]} ]}, :default, [:binary]} ]} - ]}, {nil, 0}}} + ]}, {nil, 15}}} ]}, :default, [:binary]}, {:bin_element, 15, {:integer, 15, 12}, :default, [:integer]} ]} @@ -200,7 +200,7 @@ defmodule Gradient.AstSpecifierTest do {:clause, 10, [], [], [ {:cons, 11, {:tuple, 11, [{:atom, 11, :a}, {:integer, 11, 12}]}, - {:cons, 11, {:tuple, 11, [{:atom, 11, :b}, {:atom, 11, :ok}]}, {nil, 0}}} + {:cons, 11, {:tuple, 11, [{:atom, 11, :b}, {:atom, 11, :ok}]}, {nil, 11}}} ]} ]} = tuple_in_list @@ -319,10 +319,11 @@ defmodule Gradient.AstSpecifierTest do [ {:case, 13, {:op, 13, :<, {:integer, 13, 1}, {:integer, 13, 5}}, [ - {:clause, [generated: true, location: 13], [{:atom, 0, false}], [], + {:clause, [generated: true, location: 13], + [{:atom, [generated: true, location: 13], false}], [], [{:atom, 16, :error}]}, - {:clause, [generated: true, location: 13], [{:atom, 0, true}], [], - [{:atom, 14, :ok}]} + {:clause, [generated: true, location: 13], + [{:atom, [generated: true, location: 13], true}], [], [{:atom, 14, :ok}]} ]} ]} ]} = block @@ -333,10 +334,11 @@ defmodule Gradient.AstSpecifierTest do [ {:case, 10, {:op, 10, :<, {:integer, 10, 1}, {:integer, 10, 5}}, [ - {:clause, [generated: true, location: 10], [{:atom, 0, false}], [], + {:clause, [generated: true, location: 10], + [{:atom, [generated: true, location: 10], false}], [], [{:atom, 10, :error}]}, - {:clause, [generated: true, location: 10], [{:atom, 0, true}], [], - [{:atom, 10, :ok}]} + {:clause, [generated: true, location: 10], + [{:atom, [generated: true, location: 10], true}], [], [{:atom, 10, :ok}]} ]} ]} ]} = inline @@ -347,10 +349,11 @@ defmodule Gradient.AstSpecifierTest do [ {:case, 4, {:op, 4, :<, {:integer, 4, 1}, {:integer, 4, 5}}, [ - {:clause, [generated: true, location: 4], [{:atom, 0, false}], [], + {:clause, [generated: true, location: 4], + [{:atom, [generated: true, location: 4], false}], [], [{:atom, 7, :error}]}, - {:clause, [generated: true, location: 4], [{:atom, 0, true}], [], - [{:atom, 5, :ok}]} + {:clause, [generated: true, location: 4], + [{:atom, [generated: true, location: 4], true}], [], [{:atom, 5, :ok}]} ]} ]} ]} = if_ @@ -372,14 +375,14 @@ defmodule Gradient.AstSpecifierTest do [ {:case, 3, {:atom, 3, false}, [ - {:clause, [generated: true, location: 3], [{:atom, 0, false}], [], - [{:atom, 4, :ok}]}, - {:clause, [generated: true, location: 3], [{:atom, 0, true}], [], - [{:atom, 6, :error}]} + {:clause, [generated: true, location: 3], + [{:atom, [generated: true, location: 3], false}], [], [{:atom, 4, :ok}]}, + {:clause, [generated: true, location: 3], + [{:atom, [generated: true, location: 3], true}], [], [{:atom, 6, :error}]} ]} ]} ] - } == block + } = block end test "cond conditional" do @@ -393,19 +396,19 @@ defmodule Gradient.AstSpecifierTest do [ {:case, 4, {:op, 5, :==, {:var, 5, :_a@1}, {:atom, 5, :ok}}, [ - {:clause, 5, [{:atom, 0, true}], [], [{:atom, 5, :ok}]}, - {:clause, 6, [{:atom, 0, false}], [], + {:clause, 5, [{:atom, 5, true}], [], [{:atom, 5, :ok}]}, + {:clause, 6, [{:atom, 6, false}], [], [ {:case, 6, {:op, 6, :>, {:var, 6, :_a@1}, {:integer, 6, 5}}, [ - {:clause, 6, [{:atom, 0, true}], [], [{:atom, 6, :ok}]}, - {:clause, 7, [{:atom, 0, false}], [], + {:clause, 6, [{:atom, 6, true}], [], [{:atom, 6, :ok}]}, + {:clause, 7, [{:atom, 7, false}], [], [ {:case, 7, {:atom, 7, true}, [ - {:clause, 7, [{:atom, 0, true}], [], [{:atom, 7, :error}]}, - {:clause, [generated: true, location: 7], [{:atom, 0, false}], - [], + {:clause, 7, [{:atom, 7, true}], [], [{:atom, 7, :error}]}, + {:clause, [generated: true, location: 7], + [{:atom, [generated: true, location: 7], false}], [], [ {:call, 7, {:remote, 7, {:atom, 7, :erlang}, {:atom, 7, :error}}, @@ -426,19 +429,19 @@ defmodule Gradient.AstSpecifierTest do {:match, 11, {:var, 11, :_a@1}, {:integer, 11, 5}}, {:case, 13, {:op, 14, :==, {:var, 14, :_a@1}, {:atom, 14, :ok}}, [ - {:clause, 14, [{:atom, 0, true}], [], [{:atom, 14, :ok}]}, - {:clause, 15, [{:atom, 0, false}], [], + {:clause, 14, [{:atom, 14, true}], [], [{:atom, 14, :ok}]}, + {:clause, 15, [{:atom, 15, false}], [], [ {:case, 15, {:op, 15, :>, {:var, 15, :_a@1}, {:integer, 15, 5}}, [ - {:clause, 15, [{:atom, 0, true}], [], [{:atom, 15, :ok}]}, - {:clause, 16, [{:atom, 0, false}], [], + {:clause, 15, [{:atom, 15, true}], [], [{:atom, 15, :ok}]}, + {:clause, 16, [{:atom, 16, false}], [], [ {:case, 16, {:atom, 16, true}, [ - {:clause, 16, [{:atom, 0, true}], [], [{:atom, 16, :error}]}, - {:clause, [generated: true, location: 16], [{:atom, 0, false}], - [], + {:clause, 16, [{:atom, 16, true}], [], [{:atom, 16, :error}]}, + {:clause, [generated: true, location: 16], + [{:atom, [generated: true, location: 16], false}], [], [ {:call, 16, {:remote, 16, {:atom, 16, :erlang}, {:atom, 16, :error}}, @@ -476,7 +479,7 @@ defmodule Gradient.AstSpecifierTest do [{:bin_element, 11, {:string, 11, 'error'}, :default, :default}]} ]}, {:cons, 12, {:integer, 12, 49}, - {:cons, 12, {:integer, 12, 50}, {nil, 0}}} + {:cons, 12, {:integer, 12, 50}, {nil, 12}}} ]} ]} ]} @@ -544,7 +547,7 @@ defmodule Gradient.AstSpecifierTest do [ {:bin, 7, [{:bin_element, 7, {:string, 7, 'ala'}, :default, :default}]}, {:cons, 8, {:integer, 8, 97}, - {:cons, 8, {:integer, 8, 108}, {:cons, 8, {:integer, 8, 97}, {nil, 0}}}}, + {:cons, 8, {:integer, 8, 108}, {:cons, 8, {:integer, 8, 97}, {nil, 8}}}}, {:integer, 9, 12} ]} ]} @@ -570,7 +573,7 @@ defmodule Gradient.AstSpecifierTest do :integer, 4, 2 - }, {:cons, 4, {:integer, 4, 3}, {nil, 0}}}}, + }, {:cons, 4, {:integer, 4, 3}, {nil, 4}}}}, {:fun, 4, {:clauses, [ @@ -764,13 +767,13 @@ defmodule Gradient.AstSpecifierTest do {:clause, 5, [], [], [ {:cons, 6, - {:cons, 6, {:integer, 6, 49}, {:cons, 6, {:integer, 6, 49}, {nil, 0}}}, + {:cons, 6, {:integer, 6, 49}, {:cons, 6, {:integer, 6, 49}, {nil, 6}}}, {:cons, 6, {:bin, 6, [{:bin_element, 6, {:string, 6, '12'}, :default, :default}]}, {:cons, 6, {:integer, 6, 1}, {:cons, 6, {:integer, 6, 2}, {:cons, 6, {:integer, 6, 3}, - {:cons, 6, {:call, 6, {:atom, 6, :wrap}, [{:integer, 6, 4}]}, {nil, 0}}}}}}} + {:cons, 6, {:call, 6, {:atom, 6, :wrap}, [{:integer, 6, 4}]}, {nil, 6}}}}}}} ]} ]} = list @@ -780,7 +783,7 @@ defmodule Gradient.AstSpecifierTest do [ {:cons, 10, {:var, 10, :_a@1}, {:cons, 10, {:integer, 10, 1}, - {:cons, 10, {:integer, 10, 2}, {:cons, 10, {:integer, 10, 3}, {nil, 0}}}}} + {:cons, 10, {:integer, 10, 2}, {:cons, 10, {:integer, 10, 3}, {nil, 10}}}}} ]} ]} = ht @@ -808,7 +811,8 @@ defmodule Gradient.AstSpecifierTest do [ {:case, 4, {:atom, 4, true}, [ - {:clause, [generated: true, location: 4], [{:atom, 0, false}], [], + {:clause, [generated: true, location: 4], + [{:atom, [generated: true, location: 4], false}], [], [ {:call, 7, {:remote, 7, {:atom, 7, :erlang}, {:atom, 7, :error}}, [ @@ -822,7 +826,8 @@ defmodule Gradient.AstSpecifierTest do ]} ]} ]}, - {:clause, [generated: true, location: 4], [{:atom, 0, true}], [], + {:clause, [generated: true, location: 4], + [{:atom, [generated: true, location: 4], true}], [], [ {:call, 5, {:remote, 5, {:atom, 5, :erlang}, {:atom, 5, :throw}}, [ @@ -922,7 +927,7 @@ defmodule Gradient.AstSpecifierTest do [ {:bin, 41, [{:bin_element, 41, {:string, 41, 'sample'}, :default, :default}]}, - {:cons, 41, {:atom, 41, :utf8}, {:cons, 41, {:atom, 41, :write}, {nil, 0}}} + {:cons, 41, {:atom, 41, :utf8}, {:cons, 41, {:atom, 41, :write}, {nil, 41}}} ]}}, {:try, 43, [ @@ -966,7 +971,7 @@ defmodule Gradient.AstSpecifierTest do {:call, 52, {:remote, 52, {:atom, 52, Kernel.Utils}, {:atom, 52, :raise}}, [ {:cons, 52, {:integer, 52, 49}, - {:cons, 52, {:integer, 52, 50}, {nil, 0}}} + {:cons, 52, {:integer, 52, 50}, {nil, 52}}} ]} ]}, {:integer, 53, 1} @@ -1065,7 +1070,7 @@ defmodule Gradient.AstSpecifierTest do [ {:map, 17, [ - {:map_field_exact, 17, {:atom, 17, :x}, + {:map_field_exact, 17, {:atom, [generated: true, location: 17], :x}, {:var, [generated: true, location: 17], :_@1}} ]} ], [], [{:var, [generated: true, location: 17], :_@1}]}, @@ -1166,7 +1171,6 @@ defmodule Gradient.AstSpecifierTest do [ {:clause, 7, [], [], [{:tuple, 8, [{:atom, 8, :record_ex}, {:integer, 8, 0}, {:integer, 8, 0}]}]} - # FIXME Should be a tuple with line 8, not 12. The line is taken from a token that is in another scope. Related to the cutting out tokens at the bottom ]} = empty assert {:function, 11, :init, 0, @@ -1175,14 +1179,16 @@ defmodule Gradient.AstSpecifierTest do [{:tuple, 12, [{:atom, 12, :record_ex}, {:integer, 12, 1}, {:integer, 12, 0}]}]} ]} = init + elixir_env_arg = if System.version() >= "1.13", do: :to_caller, else: :linify + assert {:function, 5, :"MACRO-record_ex", 1, [ {:clause, 5, [{:var, 5, :_@CALLER}], [], [ {:match, 5, {:var, 5, :__CALLER__}, - {:call, 5, {:remote, 5, {:atom, 5, :elixir_env}, {:atom, 5, :linify}}, + {:call, 5, {:remote, 5, {:atom, 5, :elixir_env}, {:atom, 5, ^elixir_env_arg}}, [{:var, 5, :_@CALLER}]}}, - {:call, 5, {:atom, 5, :"MACRO-record_ex"}, [{:var, 5, :__CALLER__}, {nil, 0}]} + {:call, 5, {:atom, 5, :"MACRO-record_ex"}, [{:var, 5, :__CALLER__}, {nil, 5}]} ]} ]} = macro1 @@ -1191,13 +1197,13 @@ defmodule Gradient.AstSpecifierTest do {:clause, 5, [{:var, 5, :_@CALLER}, {:var, 5, :_@1}], [], [ {:match, 5, {:var, 5, :__CALLER__}, - {:call, 5, {:remote, 5, {:atom, 5, :elixir_env}, {:atom, 5, :linify}}, + {:call, 5, {:remote, 5, {:atom, 5, :elixir_env}, {:atom, 5, ^elixir_env_arg}}, [{:var, 5, :_@CALLER}]}}, {:call, 5, {:remote, 5, {:atom, 5, Record}, {:atom, 5, :__access__}}, [ {:atom, 5, :record_ex}, {:cons, 5, {:tuple, 5, [{:atom, 5, :x}, {:integer, 5, 0}]}, - {:cons, 5, {:tuple, 5, [{:atom, 5, :y}, {:integer, 5, 0}]}, {nil, 0}}}, + {:cons, 5, {:tuple, 5, [{:atom, 5, :y}, {:integer, 5, 0}]}, {nil, 5}}}, {:var, 5, :_@1}, {:var, 5, :__CALLER__} ]} @@ -1209,13 +1215,13 @@ defmodule Gradient.AstSpecifierTest do {:clause, 5, [{:var, 5, :_@CALLER}, {:var, 5, :_@1}, {:var, 5, :_@2}], [], [ {:match, 5, {:var, 5, :__CALLER__}, - {:call, 5, {:remote, 5, {:atom, 5, :elixir_env}, {:atom, 5, :linify}}, + {:call, 5, {:remote, 5, {:atom, 5, :elixir_env}, {:atom, 5, ^elixir_env_arg}}, [{:var, 5, :_@CALLER}]}}, {:call, 5, {:remote, 5, {:atom, 5, Record}, {:atom, 5, :__access__}}, [ {:atom, 5, :record_ex}, {:cons, 5, {:tuple, 5, [{:atom, 5, :x}, {:integer, 5, 0}]}, - {:cons, 5, {:tuple, 5, [{:atom, 5, :y}, {:integer, 5, 0}]}, {nil, 0}}}, + {:cons, 5, {:tuple, 5, [{:atom, 5, :y}, {:integer, 5, 0}]}, {nil, 5}}}, {:var, 5, :_@1}, {:var, 5, :_@2}, {:var, 5, :__CALLER__}