From b032662182bbe5a8be9cdf86dcbc694bc2554f05 Mon Sep 17 00:00:00 2001 From: Felipe Renan Date: Wed, 23 Feb 2022 22:34:45 +0000 Subject: [PATCH 1/3] Refactor Eex tokenizer This commit moves the line and column to the last argument to be handled as a metadata as you can see in the example below: ```elixir assert T.tokenize('foo', 1, 1, @opts) == {:ok, [{:text, 'foo', %{line: 1, column: 1}}, {:eof, %{line: 1, column: 4}}]} ``` This also expose Eex.tokenize/2 which takes line and column as options. --- lib/eex/lib/eex/compiler.ex | 74 ++++---- lib/eex/lib/eex/tokenizer.ex | 32 ++-- lib/eex/test/eex/tokenizer_test.exs | 259 ++++++++++++++++------------ 3 files changed, 206 insertions(+), 159 deletions(-) diff --git a/lib/eex/lib/eex/compiler.ex b/lib/eex/lib/eex/compiler.ex index 2654d7bcb93..f509296ce8b 100644 --- a/lib/eex/lib/eex/compiler.ex +++ b/lib/eex/lib/eex/compiler.ex @@ -17,9 +17,9 @@ defmodule EEx.Compiler do indentation = opts[:indentation] || 0 trim = opts[:trim] || false parser_options = opts[:parser_options] || Code.get_compiler_option(:parser_options) - tokenizer_options = %{trim: trim, indentation: indentation} + tokenizer_options = %{trim: trim, indentation: indentation, line: line, column: column} - case EEx.Tokenizer.tokenize(source, line, column, tokenizer_options) do + case EEx.Tokenizer.tokenize(source, tokenizer_options) do {:ok, tokens} -> state = %{ engine: opts[:engine] || @default_engine, @@ -42,10 +42,10 @@ defmodule EEx.Compiler do # Generates the buffers by handling each expression from the tokenizer. # It returns Macro.t/0 or it raises. - defp generate_buffer([{:text, line, column, chars} | rest], buffer, scope, state) do + defp generate_buffer([{:text, chars, meta} | rest], buffer, scope, state) do buffer = if function_exported?(state.engine, :handle_text, 3) do - meta = [line: line, column: column] + meta = [line: meta.line, column: meta.column] state.engine.handle_text(buffer, meta, IO.chardata_to_string(chars)) else # TODO: Remove this branch on Elixir v2.0 @@ -55,15 +55,18 @@ defmodule EEx.Compiler do generate_buffer(rest, buffer, scope, state) end - defp generate_buffer([{:expr, line, column, mark, chars} | rest], buffer, scope, state) do - options = [file: state.file, line: line, column: column(column, mark)] ++ state.parser_options + defp generate_buffer([{:expr, mark, chars, meta} | rest], buffer, scope, state) do + options = + [file: state.file, line: meta.line, column: column(meta.column, mark)] ++ + state.parser_options + expr = Code.string_to_quoted!(chars, options) buffer = state.engine.handle_expr(buffer, IO.chardata_to_string(mark), expr) generate_buffer(rest, buffer, scope, state) end defp generate_buffer( - [{:start_expr, start_line, start_column, mark, chars} | rest], + [{:start_expr, mark, chars, meta} | rest], buffer, scope, state @@ -72,11 +75,10 @@ defmodule EEx.Compiler do message = "the contents of this expression won't be output unless the EEx block starts with \"<%=\"" - :elixir_errors.erl_warn({start_line, start_column}, state.file, message) + :elixir_errors.erl_warn({meta.line, meta.column}, state.file, message) end - {rest, line, contents} = - look_ahead_middle(rest, start_line, chars) || {rest, start_line, chars} + {rest, line, contents} = look_ahead_middle(rest, meta.line, chars) || {rest, meta.line, chars} {contents, rest} = generate_buffer( @@ -87,8 +89,8 @@ defmodule EEx.Compiler do state | quoted: [], line: line, - start_line: start_line, - start_column: column(start_column, mark) + start_line: meta.line, + start_column: column(meta.column, mark) } ) @@ -97,18 +99,18 @@ defmodule EEx.Compiler do end defp generate_buffer( - [{:middle_expr, line, _column, '', chars} | rest], + [{:middle_expr, '', chars, meta} | rest], buffer, [current | scope], state ) do - {wrapped, state} = wrap_expr(current, line, buffer, chars, state) - state = %{state | line: line} + {wrapped, state} = wrap_expr(current, meta.line, buffer, chars, state) + state = %{state | line: meta.line} generate_buffer(rest, state.engine.handle_begin(buffer), [wrapped | scope], state) end defp generate_buffer( - [{:middle_expr, line, column, modifier, chars} | t], + [{:middle_expr, modifier, chars, meta} | t], buffer, [_ | _] = scope, state @@ -117,27 +119,27 @@ defmodule EEx.Compiler do "unexpected beginning of EEx tag \"<%#{modifier}\" on \"<%#{modifier}#{chars}%>\", " <> "please remove \"#{modifier}\" accordingly" - :elixir_errors.erl_warn({line, column}, state.file, message) - generate_buffer([{:middle_expr, line, column, '', chars} | t], buffer, scope, state) + :elixir_errors.erl_warn({meta.line, meta.column}, state.file, message) + generate_buffer([{:middle_expr, '', chars, meta} | t], buffer, scope, state) # TODO: Make this an error on Elixir v2.0 since it accidentally worked previously. # raise EEx.SyntaxError, message: message, file: state.file, line: line end - defp generate_buffer([{:middle_expr, line, column, _, chars} | _], _buffer, [], state) do + defp generate_buffer([{:middle_expr, _, chars, meta} | _], _buffer, [], state) do raise EEx.SyntaxError, message: "unexpected middle of expression <%#{chars}%>", file: state.file, - line: line, - column: column + line: meta.line, + column: meta.column end defp generate_buffer( - [{:end_expr, line, _column, '', chars} | rest], + [{:end_expr, '', chars, meta} | rest], buffer, [current | _], state ) do - {wrapped, state} = wrap_expr(current, line, buffer, chars, state) + {wrapped, state} = wrap_expr(current, meta.line, buffer, chars, state) column = state.start_column options = [file: state.file, line: state.start_line, column: column] ++ state.parser_options tuples = Code.string_to_quoted!(wrapped, options) @@ -146,7 +148,7 @@ defmodule EEx.Compiler do end defp generate_buffer( - [{:end_expr, line, column, modifier, chars} | t], + [{:end_expr, modifier, chars, meta} | t], buffer, [_ | _] = scope, state @@ -155,30 +157,30 @@ defmodule EEx.Compiler do "unexpected beginning of EEx tag \"<%#{modifier}\" on end of " <> "expression \"<%#{modifier}#{chars}%>\", please remove \"#{modifier}\" accordingly" - :elixir_errors.erl_warn({line, column}, state.file, message) - generate_buffer([{:end_expr, line, column, '', chars} | t], buffer, scope, state) + :elixir_errors.erl_warn({meta.line, meta.column}, state.file, message) + generate_buffer([{:end_expr, '', chars, meta} | t], buffer, scope, state) # TODO: Make this an error on Elixir v2.0 since it accidentally worked previously. # raise EEx.SyntaxError, message: message, file: state.file, line: line, column: column end - defp generate_buffer([{:end_expr, line, column, _, chars} | _], _buffer, [], state) do + defp generate_buffer([{:end_expr, _, chars, meta} | _], _buffer, [], state) do raise EEx.SyntaxError, message: "unexpected end of expression <%#{chars}%>", file: state.file, - line: line, - column: column + line: meta.line, + column: meta.column end - defp generate_buffer([{:eof, _, _}], buffer, [], state) do + defp generate_buffer([{:eof, _meta}], buffer, [], state) do state.engine.handle_body(buffer) end - defp generate_buffer([{:eof, line, column}], _buffer, _scope, state) do + defp generate_buffer([{:eof, meta}], _buffer, _scope, state) do raise EEx.SyntaxError, message: "unexpected end of string, expected a closing '<% end %>'", file: state.file, - line: line, - column: column + line: meta.line, + column: meta.column end # Creates a placeholder and wrap it inside the expression block @@ -195,7 +197,7 @@ defmodule EEx.Compiler do # Look middle expressions that immediately follow a start_expr - defp look_ahead_middle([{:text, _, _, text} | rest], start, contents) do + defp look_ahead_middle([{:text, text, _meta} | rest], start, contents) do if only_spaces?(text) do look_ahead_middle(rest, start, contents ++ text) else @@ -203,8 +205,8 @@ defmodule EEx.Compiler do end end - defp look_ahead_middle([{:middle_expr, line, _column, _, chars} | rest], _start, contents) do - {rest, line, contents ++ chars} + defp look_ahead_middle([{:middle_expr, chars, meta} | rest], _start, contents) do + {rest, meta.line, contents ++ chars} end defp look_ahead_middle(_tokens, _start, _contents) do diff --git a/lib/eex/lib/eex/tokenizer.ex b/lib/eex/lib/eex/tokenizer.ex index fdfefe43ce8..73a798be2d4 100644 --- a/lib/eex/lib/eex/tokenizer.ex +++ b/lib/eex/lib/eex/tokenizer.ex @@ -5,10 +5,11 @@ defmodule EEx.Tokenizer do @type line :: non_neg_integer @type column :: non_neg_integer @type marker :: '=' | '/' | '|' | '' + @type metadata :: %{column: column, line: line} @type token :: - {:text, line, column, content} - | {:expr | :start_expr | :middle_expr | :end_expr, line, column, marker, content} - | {:eof, line, column} + {:text, content, metadata} + | {:expr | :start_expr | :middle_expr | :end_expr, marker, content, metadata} + | {:eof, metadata} @spaces [?\s, ?\t] @@ -17,17 +18,22 @@ defmodule EEx.Tokenizer do It returns {:ok, list} with the following tokens: - * `{:text, line, column, content}` - * `{:expr, line, column, marker, content}` - * `{:start_expr, line, column, marker, content}` - * `{:middle_expr, line, column, marker, content}` - * `{:end_expr, line, column, marker, content}` - * `{:eof, line, column}` + * `{:text, content, %{column: column, line: line}}` + * `{:expr, marker, content, %{column: column, line: line}}` + * `{:start_expr, marker, content, %{column: column, line: line}}` + * `{:middle_expr, marker, content, %{column: column, line: line}}` + * `{:end_expr, marker, content, %{column: column, line: line}}` + * `{:eof, %{column: column, line: line}}` Or `{:error, line, column, message}` in case of errors. """ - @spec tokenize(binary | charlist, line, column, map) :: + @spec tokenize(binary | charlist, map) :: {:ok, [token]} | {:error, line, column, String.t()} + def tokenize(contents, opts) do + line = opts[:line] || 1 + column = opts[:column] || 1 + tokenize(contents, line, column, opts) + end def tokenize(bin, line, column, opts) when is_binary(bin) do tokenize(String.to_charlist(bin), line, column, opts) @@ -89,7 +95,7 @@ defmodule EEx.Tokenizer do {:expr, expr} end - token = {key, line, column, marker, expr} + token = {key, marker, expr, %{line: line, column: column}} trim_and_tokenize(rest, new_line, new_column, opts, buffer, acc, &[token | &1]) end end @@ -103,7 +109,7 @@ defmodule EEx.Tokenizer do end defp tokenize([], line, column, _opts, buffer, acc) do - eof = {:eof, line, column} + eof = {:eof, %{line: line, column: column}} {:ok, Enum.reverse([eof | tokenize_text(buffer, acc)])} end @@ -212,7 +218,7 @@ defmodule EEx.Tokenizer do defp tokenize_text(buffer, acc) do [{line, column} | buffer] = Enum.reverse(buffer) - [{:text, line, column, buffer} | acc] + [{:text, buffer, %{line: line, column: column}} | acc] end ## Trim diff --git a/lib/eex/test/eex/tokenizer_test.exs b/lib/eex/test/eex/tokenizer_test.exs index 3a410fc0b53..333c4052263 100644 --- a/lib/eex/test/eex/tokenizer_test.exs +++ b/lib/eex/test/eex/tokenizer_test.exs @@ -7,36 +7,63 @@ defmodule EEx.TokenizerTest do @opts %{indentation: 0, trim: false} test "simple chars lists" do - assert T.tokenize('foo', 1, 1, @opts) == {:ok, [{:text, 1, 1, 'foo'}, {:eof, 1, 4}]} + assert T.tokenize('foo', 1, 1, @opts) == + {:ok, [{:text, 'foo', %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} end test "simple strings" do - assert T.tokenize("foo", 1, 1, @opts) == {:ok, [{:text, 1, 1, 'foo'}, {:eof, 1, 4}]} + assert T.tokenize("foo", 1, 1, @opts) == + {:ok, [{:text, 'foo', %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} end test "strings with embedded code" do assert T.tokenize('foo <% bar %>', 1, 1, @opts) == - {:ok, [{:text, 1, 1, 'foo '}, {:expr, 1, 5, '', ' bar '}, {:eof, 1, 14}]} + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 14, line: 1}} + ]} end test "strings with embedded equals code" do assert T.tokenize('foo <%= bar %>', 1, 1, @opts) == - {:ok, [{:text, 1, 1, 'foo '}, {:expr, 1, 5, '=', ' bar '}, {:eof, 1, 15}]} + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '=', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} end test "strings with embedded slash code" do assert T.tokenize('foo <%/ bar %>', 1, 1, @opts) == - {:ok, [{:text, 1, 1, 'foo '}, {:expr, 1, 5, '/', ' bar '}, {:eof, 1, 15}]} + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '/', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} end test "strings with embedded pipe code" do assert T.tokenize('foo <%| bar %>', 1, 1, @opts) == - {:ok, [{:text, 1, 1, 'foo '}, {:expr, 1, 5, '|', ' bar '}, {:eof, 1, 15}]} + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '|', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} end test "strings with more than one line" do assert T.tokenize('foo\n<%= bar %>', 1, 1, @opts) == - {:ok, [{:text, 1, 1, 'foo\n'}, {:expr, 2, 1, '=', ' bar '}, {:eof, 2, 11}]} + {:ok, + [ + {:text, 'foo\n', %{column: 1, line: 1}}, + {:expr, '=', ' bar ', %{column: 1, line: 2}}, + {:eof, %{column: 11, line: 2}} + ]} end test "strings with more than one line and expression with more than one line" do @@ -48,12 +75,12 @@ defmodule EEx.TokenizerTest do ''' exprs = [ - {:text, 1, 1, 'foo '}, - {:expr, 1, 5, '=', ' bar\n\nbaz '}, - {:text, 3, 7, '\n'}, - {:expr, 4, 1, '', ' foo '}, - {:text, 4, 10, '\n'}, - {:eof, 5, 1} + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '=', ' bar\n\nbaz ', %{column: 5, line: 1}}, + {:text, '\n', %{column: 7, line: 3}}, + {:expr, '', ' foo ', %{column: 1, line: 4}}, + {:text, '\n', %{column: 10, line: 4}}, + {:eof, %{column: 1, line: 5}} ] assert T.tokenize(string, 1, 1, @opts) == {:ok, exprs} @@ -61,22 +88,30 @@ defmodule EEx.TokenizerTest do test "quotation" do assert T.tokenize('foo <%% true %>', 1, 1, @opts) == - {:ok, [{:text, 1, 1, 'foo <% true %>'}, {:eof, 1, 16}]} + {:ok, + [ + {:text, 'foo <% true %>', %{column: 1, line: 1}}, + {:eof, %{column: 16, line: 1}} + ]} end test "quotation with do-end" do assert T.tokenize('foo <%% true do %>bar<%% end %>', 1, 1, @opts) == - {:ok, [{:text, 1, 1, 'foo <% true do %>bar<% end %>'}, {:eof, 1, 32}]} + {:ok, + [ + {:text, 'foo <% true do %>bar<% end %>', %{column: 1, line: 1}}, + {:eof, %{column: 32, line: 1}} + ]} end test "quotation with interpolation" do exprs = [ - {:text, 1, 1, 'a <% b '}, - {:expr, 1, 9, '=', ' c '}, - {:text, 1, 17, ' '}, - {:expr, 1, 18, '=', ' d '}, - {:text, 1, 26, ' e %> f'}, - {:eof, 1, 33} + {:text, 'a <% b ', %{column: 1, line: 1}}, + {:expr, '=', ' c ', %{column: 9, line: 1}}, + {:text, ' ', %{column: 17, line: 1}}, + {:expr, '=', ' d ', %{column: 18, line: 1}}, + {:text, ' e %> f', %{column: 26, line: 1}}, + {:eof, %{column: 33, line: 1}} ] assert T.tokenize('a <%% b <%= c %> <%= d %> e %> f', 1, 1, @opts) == {:ok, exprs} @@ -84,8 +119,8 @@ defmodule EEx.TokenizerTest do test "improperly formatted quotation with interpolation" do exprs = [ - {:text, 1, 1, '<%% a <%= b %> c %>'}, - {:eof, 1, 22} + {:text, '<%% a <%= b %> c %>', %{column: 1, line: 1}}, + {:eof, %{column: 22, line: 1}} ] assert T.tokenize('<%%% a <%%= b %> c %>', 1, 1, @opts) == {:ok, exprs} @@ -93,15 +128,15 @@ defmodule EEx.TokenizerTest do test "EEx comments" do exprs = [ - {:text, 1, 1, 'foo '}, - {:eof, 1, 16} + {:text, 'foo ', %{column: 1, line: 1}}, + {:eof, %{column: 16, line: 1}} ] assert T.tokenize('foo <%# true %>', 1, 1, @opts) == {:ok, exprs} exprs = [ - {:text, 1, 1, 'foo '}, - {:eof, 2, 8} + {:text, 'foo ', %{column: 1, line: 1}}, + {:eof, %{column: 8, line: 2}} ] assert T.tokenize('foo <%#\ntrue %>', 1, 1, @opts) == {:ok, exprs} @@ -109,9 +144,9 @@ defmodule EEx.TokenizerTest do test "EEx comments with do-end" do exprs = [ - {:text, 1, 1, 'foo '}, - {:text, 1, 19, 'bar'}, - {:eof, 1, 32} + {:text, 'foo ', %{column: 1, line: 1}}, + {:text, 'bar', %{column: 19, line: 1}}, + {:eof, %{column: 32, line: 1}} ] assert T.tokenize('foo <%# true do %>bar<%# end %>', 1, 1, @opts) == {:ok, exprs} @@ -119,20 +154,20 @@ defmodule EEx.TokenizerTest do test "EEx comments inside do-end" do exprs = [ - {:start_expr, 1, 1, '', ' if true do '}, - {:text, 1, 31, 'bar'}, - {:end_expr, 1, 34, [], ' end '}, - {:eof, 1, 43} + {:start_expr, '', ' if true do ', %{column: 1, line: 1}}, + {:text, 'bar', %{column: 31, line: 1}}, + {:end_expr, [], ' end ', %{column: 34, line: 1}}, + {:eof, %{column: 43, line: 1}} ] assert T.tokenize('<% if true do %><%# comment %>bar<% end %>', 1, 1, @opts) == {:ok, exprs} exprs = [ - {:start_expr, 1, 1, [], ' case true do '}, - {:middle_expr, 1, 33, '', ' true -> '}, - {:text, 1, 46, 'bar'}, - {:end_expr, 1, 49, [], ' end '}, - {:eof, 1, 58} + {:start_expr, [], ' case true do ', %{column: 1, line: 1}}, + {:middle_expr, '', ' true -> ', %{column: 33, line: 1}}, + {:text, 'bar', %{column: 46, line: 1}}, + {:end_expr, [], ' end ', %{column: 49, line: 1}}, + {:eof, %{column: 58, line: 1}} ] assert T.tokenize('<% case true do %><%# comment %><% true -> %>bar<% end %>', 1, 1, @opts) == @@ -141,25 +176,25 @@ defmodule EEx.TokenizerTest do test "EEx multi-line comments" do exprs = [ - {:text, 1, 1, 'foo '}, - {:text, 1, 20, ' bar'}, - {:eof, 1, 24} + {:text, 'foo ', %{column: 1, line: 1}}, + {:text, ' bar', %{column: 20, line: 1}}, + {:eof, %{column: 24, line: 1}} ] assert T.tokenize('foo <%!-- true --%> bar', 1, 1, @opts) == {:ok, exprs} exprs = [ - {:text, 1, 1, 'foo '}, - {:text, 3, 6, ' bar'}, - {:eof, 3, 10} + {:text, 'foo ', %{column: 1, line: 1}}, + {:text, ' bar', %{column: 6, line: 3}}, + {:eof, %{column: 10, line: 3}} ] assert T.tokenize('foo <%!-- \ntrue\n --%> bar', 1, 1, @opts) == {:ok, exprs} exprs = [ - {:text, 1, 1, 'foo '}, - {:text, 1, 27, ' bar'}, - {:eof, 1, 31} + {:text, 'foo ', %{column: 1, line: 1}}, + {:text, ' bar', %{column: 27, line: 1}}, + {:eof, %{column: 31, line: 1}} ] assert T.tokenize('foo <%!-- <%= true %> --%> bar', 1, 1, @opts) == {:ok, exprs} @@ -167,9 +202,9 @@ defmodule EEx.TokenizerTest do test "Elixir comments" do exprs = [ - {:text, 1, 1, 'foo '}, - {:expr, 1, 5, [], ' true # this is a boolean '}, - {:eof, 1, 35} + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, [], ' true # this is a boolean ', %{column: 5, line: 1}}, + {:eof, %{column: 35, line: 1}} ] assert T.tokenize('foo <% true # this is a boolean %>', 1, 1, @opts) == {:ok, exprs} @@ -177,10 +212,10 @@ defmodule EEx.TokenizerTest do test "Elixir comments with do-end" do exprs = [ - {:start_expr, 1, 1, [], ' if true do # startif '}, - {:text, 1, 27, 'text'}, - {:end_expr, 1, 31, [], ' end # closeif '}, - {:eof, 1, 50} + {:start_expr, [], ' if true do # startif ', %{column: 1, line: 1}}, + {:text, 'text', %{column: 27, line: 1}}, + {:end_expr, [], ' end # closeif ', %{column: 31, line: 1}}, + {:eof, %{column: 50, line: 1}} ] assert T.tokenize('<% if true do # startif %>text<% end # closeif %>', 1, 1, @opts) == @@ -189,11 +224,11 @@ defmodule EEx.TokenizerTest do test "strings with embedded do end" do exprs = [ - {:text, 1, 1, 'foo '}, - {:start_expr, 1, 5, '', ' if true do '}, - {:text, 1, 21, 'bar'}, - {:end_expr, 1, 24, '', ' end '}, - {:eof, 1, 33} + {:text, 'foo ', %{column: 1, line: 1}}, + {:start_expr, '', ' if true do ', %{column: 5, line: 1}}, + {:text, 'bar', %{column: 21, line: 1}}, + {:end_expr, '', ' end ', %{column: 24, line: 1}}, + {:eof, %{column: 33, line: 1}} ] assert T.tokenize('foo <% if true do %>bar<% end %>', 1, 1, @opts) == {:ok, exprs} @@ -201,14 +236,14 @@ defmodule EEx.TokenizerTest do test "strings with embedded -> end" do exprs = [ - {:text, 1, 1, 'foo '}, - {:start_expr, 1, 5, '', ' cond do '}, - {:middle_expr, 1, 18, '', ' false -> '}, - {:text, 1, 32, 'bar'}, - {:middle_expr, 1, 35, '', ' true -> '}, - {:text, 1, 48, 'baz'}, - {:end_expr, 1, 51, '', ' end '}, - {:eof, 1, 60} + {:text, 'foo ', %{column: 1, line: 1}}, + {:start_expr, '', ' cond do ', %{column: 5, line: 1}}, + {:middle_expr, '', ' false -> ', %{column: 18, line: 1}}, + {:text, 'bar', %{column: 32, line: 1}}, + {:middle_expr, '', ' true -> ', %{column: 35, line: 1}}, + {:text, 'baz', %{column: 48, line: 1}}, + {:end_expr, '', ' end ', %{column: 51, line: 1}}, + {:eof, %{column: 60, line: 1}} ] assert T.tokenize('foo <% cond do %><% false -> %>bar<% true -> %>baz<% end %>', 1, 1, @opts) == @@ -217,10 +252,10 @@ defmodule EEx.TokenizerTest do test "strings with fn-end with newline" do exprs = [ - {:start_expr, 1, 1, '=', ' a fn ->\n'}, - {:text, 2, 3, 'foo'}, - {:end_expr, 2, 6, [], ' end '}, - {:eof, 2, 15} + {:start_expr, '=', ' a fn ->\n', %{column: 1, line: 1}}, + {:text, 'foo', %{column: 3, line: 2}}, + {:end_expr, [], ' end ', %{column: 6, line: 2}}, + {:eof, %{column: 15, line: 2}} ] assert T.tokenize('<%= a fn ->\n%>foo<% end %>', 1, 1, @opts) == @@ -229,12 +264,12 @@ defmodule EEx.TokenizerTest do test "strings with multiple fn-end" do exprs = [ - {:start_expr, 1, 1, '=', ' a fn -> '}, - {:text, 1, 15, 'foo'}, - {:middle_expr, 1, 18, '', ' end, fn -> '}, - {:text, 1, 34, 'bar'}, - {:end_expr, 1, 37, '', ' end '}, - {:eof, 1, 46} + {:start_expr, '=', ' a fn -> ', %{column: 1, line: 1}}, + {:text, 'foo', %{column: 15, line: 1}}, + {:middle_expr, '', ' end, fn -> ', %{column: 18, line: 1}}, + {:text, 'bar', %{column: 34, line: 1}}, + {:end_expr, '', ' end ', %{column: 37, line: 1}}, + {:eof, %{column: 46, line: 1}} ] assert T.tokenize('<%= a fn -> %>foo<% end, fn -> %>bar<% end %>', 1, 1, @opts) == @@ -243,12 +278,12 @@ defmodule EEx.TokenizerTest do test "strings with fn-end followed by do block" do exprs = [ - {:start_expr, 1, 1, '=', ' a fn -> '}, - {:text, 1, 15, 'foo'}, - {:middle_expr, 1, 18, '', ' end do '}, - {:text, 1, 30, 'bar'}, - {:end_expr, 1, 33, '', ' end '}, - {:eof, 1, 42} + {:start_expr, '=', ' a fn -> ', %{column: 1, line: 1}}, + {:text, 'foo', %{column: 15, line: 1}}, + {:middle_expr, '', ' end do ', %{column: 18, line: 1}}, + {:text, 'bar', %{column: 30, line: 1}}, + {:end_expr, '', ' end ', %{column: 33, line: 1}}, + {:eof, %{column: 42, line: 1}} ] assert T.tokenize('<%= a fn -> %>foo<% end do %>bar<% end %>', 1, 1, @opts) == {:ok, exprs} @@ -256,13 +291,13 @@ defmodule EEx.TokenizerTest do test "strings with embedded keywords blocks" do exprs = [ - {:text, 1, 1, 'foo '}, - {:start_expr, 1, 5, '', ' if true do '}, - {:text, 1, 21, 'bar'}, - {:middle_expr, 1, 24, '', ' else '}, - {:text, 1, 34, 'baz'}, - {:end_expr, 1, 37, '', ' end '}, - {:eof, 1, 46} + {:text, 'foo ', %{column: 1, line: 1}}, + {:start_expr, '', ' if true do ', %{column: 5, line: 1}}, + {:text, 'bar', %{column: 21, line: 1}}, + {:middle_expr, '', ' else ', %{column: 24, line: 1}}, + {:text, 'baz', %{column: 34, line: 1}}, + {:end_expr, '', ' end ', %{column: 37, line: 1}}, + {:eof, %{column: 46, line: 1}} ] assert T.tokenize('foo <% if true do %>bar<% else %>baz<% end %>', 1, 1, @opts) == @@ -273,12 +308,12 @@ defmodule EEx.TokenizerTest do template = '\t<%= if true do %> \n TRUE \n <% else %>\n FALSE \n <% end %> \n\n ' exprs = [ - {:start_expr, 1, 2, '=', ' if true do '}, - {:text, 1, 20, '\n TRUE \n'}, - {:middle_expr, 3, 3, '', ' else '}, - {:text, 3, 13, '\n FALSE \n'}, - {:end_expr, 5, 3, '', ' end '}, - {:eof, 7, 3} + {:start_expr, '=', ' if true do ', %{column: 2, line: 1}}, + {:text, '\n TRUE \n', %{column: 20, line: 1}}, + {:middle_expr, '', ' else ', %{column: 3, line: 3}}, + {:text, '\n FALSE \n', %{column: 13, line: 3}}, + {:end_expr, '', ' end ', %{column: 3, line: 5}}, + {:eof, %{column: 3, line: 7}} ] assert T.tokenize(template, 1, 1, %{@opts | trim: true}) == {:ok, exprs} @@ -286,8 +321,8 @@ defmodule EEx.TokenizerTest do test "trim mode with comment" do exprs = [ - {:text, 1, 19, '\n123'}, - {:eof, 2, 4} + {:text, '\n123', %{column: 19, line: 1}}, + {:eof, %{column: 4, line: 2}} ] assert T.tokenize(' <%# comment %> \n123', 1, 1, %{@opts | trim: true}) == {:ok, exprs} @@ -295,8 +330,8 @@ defmodule EEx.TokenizerTest do test "trim mode with multi-line comment" do exprs = [ - {:text, 1, 23, '\n123'}, - {:eof, 2, 4} + {:text, '\n123', %{column: 23, line: 1}}, + {:eof, %{column: 4, line: 2}} ] assert T.tokenize(' <%!-- comment --%> \n123', 1, 1, %{@opts | trim: true}) == {:ok, exprs} @@ -304,10 +339,10 @@ defmodule EEx.TokenizerTest do test "trim mode with CRLF" do exprs = [ - {:text, 1, 1, '0\n'}, - {:expr, 2, 3, '=', ' 12 '}, - {:text, 2, 15, '\n34'}, - {:eof, 3, 3} + {:text, '0\n', %{column: 1, line: 1}}, + {:expr, '=', ' 12 ', %{column: 3, line: 2}}, + {:text, '\n34', %{column: 15, line: 2}}, + {:eof, %{column: 3, line: 3}} ] assert T.tokenize('0\r\n <%= 12 %> \r\n34', 1, 1, %{@opts | trim: true}) == {:ok, exprs} @@ -315,10 +350,10 @@ defmodule EEx.TokenizerTest do test "trim mode set to false" do exprs = [ - {:text, 1, 1, ' '}, - {:expr, 1, 2, '=', ' 12 '}, - {:text, 1, 11, ' \n'}, - {:eof, 2, 1} + {:text, ' ', %{column: 1, line: 1}}, + {:expr, '=', ' 12 ', %{column: 2, line: 1}}, + {:text, ' \n', %{column: 11, line: 1}}, + {:eof, %{column: 1, line: 2}} ] assert T.tokenize(' <%= 12 %> \n', 1, 1, %{@opts | trim: false}) == {:ok, exprs} @@ -342,6 +377,10 @@ defmodule EEx.TokenizerTest do test "marks invalid expressions as regular expressions" do assert T.tokenize('<% 1 $ 2 %>', 1, 1, @opts) == - {:ok, [{:expr, 1, 1, [], ' 1 $ 2 '}, {:eof, 1, 12}]} + {:ok, + [ + {:expr, [], ' 1 $ 2 ', %{column: 1, line: 1}}, + {:eof, %{column: 12, line: 1}} + ]} end end From 3fc6215ba969b49b72204500441064330d14a498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 25 Feb 2022 11:09:16 +0100 Subject: [PATCH 2/3] Update lib/eex/lib/eex/compiler.ex --- lib/eex/lib/eex/compiler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/eex/lib/eex/compiler.ex b/lib/eex/lib/eex/compiler.ex index f509296ce8b..bedb63fcc87 100644 --- a/lib/eex/lib/eex/compiler.ex +++ b/lib/eex/lib/eex/compiler.ex @@ -205,7 +205,7 @@ defmodule EEx.Compiler do end end - defp look_ahead_middle([{:middle_expr, chars, meta} | rest], _start, contents) do + defp look_ahead_middle([{:middle_expr, _, chars, meta} | rest], _start, contents) do {rest, meta.line, contents ++ chars} end From 63ba841ad86e58dd146c954b6220b3c92a987c0f Mon Sep 17 00:00:00 2001 From: Felipe Renan Date: Fri, 25 Feb 2022 15:35:20 +0000 Subject: [PATCH 3/3] Normalize error messages --- lib/eex/lib/eex/compiler.ex | 2 +- lib/eex/lib/eex/tokenizer.ex | 14 +++++++------- lib/eex/test/eex/tokenizer_test.exs | 7 +++++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/eex/lib/eex/compiler.ex b/lib/eex/lib/eex/compiler.ex index bedb63fcc87..4f5e1ea48fa 100644 --- a/lib/eex/lib/eex/compiler.ex +++ b/lib/eex/lib/eex/compiler.ex @@ -34,7 +34,7 @@ defmodule EEx.Compiler do init = state.engine.init(opts) generate_buffer(tokens, init, [], state) - {:error, line, column, message} -> + {:error, message, %{column: column, line: line}} -> raise EEx.SyntaxError, file: file, line: line, column: column, message: message end end diff --git a/lib/eex/lib/eex/tokenizer.ex b/lib/eex/lib/eex/tokenizer.ex index 73a798be2d4..650e08a9b1e 100644 --- a/lib/eex/lib/eex/tokenizer.ex +++ b/lib/eex/lib/eex/tokenizer.ex @@ -25,7 +25,7 @@ defmodule EEx.Tokenizer do * `{:end_expr, marker, content, %{column: column, line: line}}` * `{:eof, %{column: column, line: line}}` - Or `{:error, line, column, message}` in case of errors. + Or `{:error, message, %{column: column, line: line}}` in case of errors. """ @spec tokenize(binary | charlist, map) :: {:ok, [token]} | {:error, line, column, String.t()} @@ -55,8 +55,8 @@ defmodule EEx.Tokenizer do defp tokenize('<%!--' ++ t, line, column, opts, buffer, acc) do case comment(t, line, column + 5, opts) do - {:error, _, _, _} = error -> - error + {:error, line, column, message} -> + {:error, message, %{line: line, column: column}} {:ok, new_line, new_column, rest} -> trim_and_tokenize(rest, new_line, new_column, opts, buffer, acc, & &1) @@ -66,8 +66,8 @@ defmodule EEx.Tokenizer do # TODO: Deprecate this on Elixir v1.18 defp tokenize('<%#' ++ t, line, column, opts, buffer, acc) do case expr(t, line, column + 3, opts, []) do - {:error, _, _, _} = error -> - error + {:error, line, column, message} -> + {:error, message, %{line: line, column: column}} {:ok, _, new_line, new_column, rest} -> trim_and_tokenize(rest, new_line, new_column, opts, buffer, acc, & &1) @@ -78,8 +78,8 @@ defmodule EEx.Tokenizer do {marker, t} = retrieve_marker(t) case expr(t, line, column + 2 + length(marker), opts, []) do - {:error, _, _, _} = error -> - error + {:error, line, column, message} -> + {:error, message, %{line: line, column: column}} {:ok, expr, new_line, new_column, rest} -> {key, expr} = diff --git a/lib/eex/test/eex/tokenizer_test.exs b/lib/eex/test/eex/tokenizer_test.exs index 333c4052263..b4436ef35ec 100644 --- a/lib/eex/test/eex/tokenizer_test.exs +++ b/lib/eex/test/eex/tokenizer_test.exs @@ -371,8 +371,11 @@ defmodule EEx.TokenizerTest do end test "returns error when there is start mark and no end mark" do - assert T.tokenize('foo <% :bar', 1, 1, @opts) == {:error, 1, 12, "missing token '%>'"} - assert T.tokenize('<%# true ', 1, 1, @opts) == {:error, 1, 10, "missing token '%>'"} + assert T.tokenize('foo <% :bar', 1, 1, @opts) == + {:error, "missing token '%>'", %{column: 12, line: 1}} + + assert T.tokenize('<%# true ', 1, 1, @opts) == + {:error, "missing token '%>'", %{column: 10, line: 1}} end test "marks invalid expressions as regular expressions" do