diff --git a/lib/gradient/ast_specifier.ex b/lib/gradient/ast_specifier.ex index fea2a438..585c2881 100644 --- a/lib/gradient/ast_specifier.ex +++ b/lib/gradient/ast_specifier.ex @@ -5,43 +5,6 @@ defmodule Gradient.AstSpecifier do it to forms that cannot be produced from Elixir directly. FIXME Optimize tokens searching. Find out why some tokens are dropped - - NOTE Mapper implements: - - function [x] - - fun [x] - - fun @spec [x] - - clause [x] - - case [x] - - block [X] - - pipe [x] - - call [x] (remote [X]) - - match [x] - - op [x] - - integer [x] - - float [x] - - string [x] - - charlist [x] - - tuple [X] - - var [X] - - list [X] - - keyword [X] - - binary [X] - - map [X] - - try [x] - - receive [X] - - record [X] elixir don't use it record_field, record_index, record_pattern, record - - named_fun [ ] is named_fun used by elixir? - - NOTE Elixir expressions to handle or test: - - list comprehension [X] - - binary [X] - - maps [X] - - struct [X] - - pipe [ ] TODO decide how to search for line in reversed form order - - range [X] - - receive [X] - - record [X] - - guards [X] """ import Gradient.Tokens @@ -55,6 +18,10 @@ defmodule Gradient.AstSpecifier do @type form :: Types.form() @type forms :: Types.forms() @type options :: Types.options() + @type abstract_expr :: Types.abstract_expr() + + # Expressions that could have missing location + @lineless_forms [:atom, :char, :float, :integer, :string, :bin, :cons, :tuple] # Api @@ -361,10 +328,9 @@ defmodule Gradient.AstSpecifier do def mapper({:call, anno, name, args}, tokens, opts) do # anno has correct line {:ok, _, anno, opts, _} = get_line(anno, opts) - name = remote_mapper(name) - {args, tokens} = context_mapper_fold(args, tokens, opts) + {args, tokens} = call_args_mapper(args, tokens, name, opts) {:call, anno, name, args} |> pass_tokens(tokens) @@ -419,8 +385,7 @@ defmodule Gradient.AstSpecifier do end def mapper({type, anno, value}, tokens, opts) - when type in [:atom, :char, :float, :integer, :string, :bin] do - # TODO check what happened for :string + when type in @lineless_forms do {:ok, line} = Keyword.fetch(opts, :line) anno = :erl_anno.set_line(line, anno) anno = :erl_anno.set_generated(Keyword.get(opts, :generated, false), anno) @@ -558,8 +523,7 @@ defmodule Gradient.AstSpecifier do @spec map_element_mapper(tuple(), tokens(), options()) :: {tuple(), tokens()} def map_element_mapper({field, anno, key, value}, tokens, opts) when field in [:map_field_assoc, :map_field_exact] do - line = :erl_anno.line(anno) - opts = Keyword.put(opts, :line, line) + {:ok, _, anno, opts, _} = get_line(anno, opts) {key, tokens} = mapper(key, tokens, opts) {value, tokens} = mapper(value, tokens, opts) @@ -625,6 +589,29 @@ defmodule Gradient.AstSpecifier do end end + @doc """ + Update location in call args with the support to the pipe operator. + """ + @spec call_args_mapper([abstract_expr()], tokens(), abstract_expr(), options()) :: + {options, [abstract_expr]} + def call_args_mapper(args, tokens, name, opts) do + # Check whether the call is after |> operator. If true, the parent location is set to 0 + # and the first arg location is cleared (if this arg is a lineless form). + # NOTE If the call is to function from :erlang module then the first arg is swapped + # with the second one because in Erlang the data is mostly in the second place. + with true <- is_pipe_op?(tokens, opts), + swapped? <- is_call_to_erlang?(name), + [fst_arg | tail_args] <- maybe_swap_args(swapped?, args), + true <- is_lineless?(fst_arg) do + {arg, tokens} = mapper(clear_location(fst_arg), tokens, Keyword.put(opts, :line, 0)) + {args, tokens} = context_mapper_fold(tail_args, tokens, opts) + {maybe_swap_args(swapped?, [arg | args]), tokens} + else + _ -> + context_mapper_fold(args, tokens, opts) + end + end + # Private Helpers @spec match_token_to_form(token(), form()) :: boolean() @@ -820,4 +807,23 @@ defmodule Gradient.AstSpecifier do defp pass_tokens(form, tokens) do {form, tokens} end + + defp is_pipe_op?(tokens, opts) do + case List.first(drop_tokens_to_line(tokens, Keyword.fetch!(opts, :line))) do + {:arrow_op, _, :|>} -> true + _ -> false + end + end + + defp maybe_swap_args(true, [fst, snd | t]), do: [snd, fst | t] + defp maybe_swap_args(_, args), do: args + + defp is_call_to_erlang?({:remote, _, {:atom, _, :erlang}, _}), do: true + defp is_call_to_erlang?(_), do: false + + defp is_lineless?(expr) do + elem(expr, 0) in @lineless_forms + end + + defp clear_location(arg), do: :erl_parse.map_anno(&:erl_anno.set_line(0, &1), arg) end diff --git a/test/gradient/ast_specifier_test.exs b/test/gradient/ast_specifier_test.exs index c85e14f3..0d47d76d 100644 --- a/test/gradient/ast_specifier_test.exs +++ b/test/gradient/ast_specifier_test.exs @@ -3,6 +3,7 @@ defmodule Gradient.AstSpecifierTest do doctest Gradient.AstSpecifier alias Gradient.AstSpecifier + alias Gradient.AstData import Gradient.TestHelpers @@ -10,6 +11,19 @@ defmodule Gradient.AstSpecifierTest do {:ok, state} end + describe "specifying expression" do + for {name, args, expected} <- AstData.ast_data() do + test "#{name}" do + {ast, tokens, opts} = unquote(Macro.escape(args)) + expected = AstData.normalize_expression(unquote(Macro.escape(expected))) + + actual = AstData.normalize_expression(elem(AstSpecifier.mapper(ast, tokens, opts), 0)) + + assert expected == actual + end + end + end + describe "run_mappers/2" do test "messy test on simple_app" do {tokens, ast} = example_data() @@ -566,13 +580,13 @@ defmodule Gradient.AstSpecifierTest do [ {:call, 4, {:remote, 4, {:atom, 4, Enum}, {:atom, 4, :filter}}, [ - {:cons, 4, {:integer, 4, 1}, - {:cons, 4, + {:cons, 3, {:integer, 3, 1}, + {:cons, 3, { :integer, - 4, + 3, 2 - }, {:cons, 4, {:integer, 4, 3}, {nil, 4}}}}, + }, {:cons, 3, {:integer, 3, 3}, {nil, 3}}}}, {:fun, 4, {:clauses, [ diff --git a/test/support/ast_data.ex b/test/support/ast_data.ex new file mode 100644 index 00000000..6fccaa9e --- /dev/null +++ b/test/support/ast_data.ex @@ -0,0 +1,165 @@ +defmodule Gradient.AstData do + @moduledoc """ + Stores the test cases data for expressions line specifying. To increase the flexibility + the data need normalization before equality assertion. Thus we check only the line change, + not the exact value and there is no need to update expected values when the file content + changes. + + This way of testing is useful only for more complex expressions in which we can observe + some line change. For example, look at the pipe operator cases. + """ + + require Gradient.Debug + import Gradient.Debug, only: [elixir_to_ast: 1] + import Gradient.TestHelpers + alias Gradient.Types + + @tokens __ENV__.file |> load_tokens() + + defp pipe do + {__ENV__.function, + {__ENV__.line, + elixir_to_ast do + 1 + |> is_atom() + + '1' + |> is_atom() + + :ok + |> is_atom() + + [1, 2, 3] + |> is_atom() + + {1, 2, 3} + |> is_atom() + + "a" + |> is_atom() + end, __ENV__.line}, + {:block, 22, + [ + {:call, 24, {:remote, 24, {:atom, 24, :erlang}, {:atom, 24, :is_atom}}, + [{:integer, 23, 1}]}, + {:call, 27, {:remote, 27, {:atom, 27, :erlang}, {:atom, 27, :is_atom}}, + [{:cons, 26, {:integer, 26, 49}, {nil, 26}}]}, + {:call, 30, {:remote, 30, {:atom, 30, :erlang}, {:atom, 30, :is_atom}}, + [{:atom, 29, :ok}]}, + {:call, 33, {:remote, 33, {:atom, 33, :erlang}, {:atom, 33, :is_atom}}, + [ + {:cons, 32, {:integer, 32, 1}, + {:cons, 32, {:integer, 32, 2}, {:cons, 32, {:integer, 32, 3}, {nil, 32}}}} + ]}, + {:call, 36, {:remote, 36, {:atom, 36, :erlang}, {:atom, 36, :is_atom}}, + [{:tuple, 35, [{:integer, 35, 1}, {:integer, 35, 2}, {:integer, 35, 3}]}]}, + {:call, 39, {:remote, 39, {:atom, 39, :erlang}, {:atom, 39, :is_atom}}, + [{:bin, 38, [{:bin_element, 38, {:string, 38, 'a'}, :default, :default}]}]} + ]}} + end + + defp pipe_with_fun_converted_to_erl_equivalent do + {__ENV__.function, + {__ENV__.line, + elixir_to_ast do + :ok + |> elem(0) + end, __ENV__.line}, + {:call, 56, {:remote, 56, {:atom, 56, :erlang}, {:atom, 56, :element}}, + [{:integer, 56, 1}, {:atom, 55, :ok}]}} + end + + defp complex_list_pipe do + {__ENV__.function, + {__ENV__.line, + elixir_to_ast do + [ + {1, %{a: 1}}, + {2, %{a: 2}} + ] + |> Enum.map(&elem(&1, 0)) + end, __ENV__.line}, + {:call, 80, {:remote, 80, {:atom, 80, Enum}, {:atom, 80, :map}}, + [ + {:cons, 76, + {:tuple, 77, + [ + {:integer, 77, 1}, + {:map, 77, [{:map_field_assoc, 77, {:atom, 77, :a}, {:integer, 77, 1}}]} + ]}, + {:cons, 77, + {:tuple, 78, + [ + {:integer, 78, 2}, + {:map, 78, [{:map_field_assoc, 78, {:atom, 78, :a}, {:integer, 78, 2}}]} + ]}, {nil, 77}}}, + {:fun, 80, + {:clauses, + [ + {:clause, 80, [{:var, 0, :_@1}], [], + [ + {:call, 80, {:remote, 80, {:atom, 80, :erlang}, {:atom, 80, :element}}, + [{:integer, 80, 1}, {:var, 0, :_@1}]} + ]} + ]}} + ]}} + end + + defp complex_tuple_pipe do + {__ENV__.function, + {__ENV__.line, + elixir_to_ast do + { + {1, %{a: 1}}, + {2, %{a: 2}} + } + |> Tuple.to_list() + end, __ENV__.line}, + {:call, 119, {:remote, 119, {:atom, 119, :erlang}, {:atom, 119, :tuple_to_list}}, + [ + {:tuple, 115, + [ + {:tuple, 116, + [ + {:integer, 116, 1}, + {:map, 116, [{:map_field_assoc, 116, {:atom, 116, :a}, {:integer, 116, 1}}]} + ]}, + {:tuple, 117, + [ + {:integer, 117, 2}, + {:map, 117, [{:map_field_assoc, 117, {:atom, 117, :a}, {:integer, 117, 2}}]} + ]} + ]} + ]}} + end + + @spec ast_data() :: [ + {atom(), {Types.abstract_expr(), Types.tokens(), Types.options()}, + Types.abstract_expr()} + ] + def ast_data do + [ + pipe(), + pipe_with_fun_converted_to_erl_equivalent(), + complex_list_pipe(), + complex_tuple_pipe() + ] + |> Enum.map(fn {{name, _}, {start_line, ast, end_line}, expected} -> + tokens = Gradient.Tokens.drop_tokens_to_line(@tokens, start_line + 1) + {name, {ast, tokens, [line: start_line + 1, end_line: end_line]}, expected} + end) + end + + def normalize_expression(expression) do + {expression, _} = + :erl_parse.mapfold_anno( + fn anno, acc -> + {{:erl_anno.line(anno) - acc, :erl_anno.column(anno)}, acc} + end, + :erl_anno.line(elem(expression, 1)), + expression + ) + + expression + end +end diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 43ba48e8..cccbbd67 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -29,6 +29,13 @@ defmodule Gradient.TestHelpers do {tokens, ast} end + def load_tokens(path) do + with {:ok, code} <- File.read(path), + {:ok, tokens} <- :elixir.string_to_tokens(String.to_charlist(code), 1, 1, path, []) do + tokens + end + end + @spec example_data() :: {T.tokens(), T.forms()} def example_data() do beam_path = Path.join(@examples_build_path, "Elixir.SimpleApp.beam") |> String.to_charlist()