From 22bc1a13bfc491eaf5fda38d87d7e35f8182764a Mon Sep 17 00:00:00 2001 From: Vladimir Dementyev Date: Thu, 23 Apr 2020 11:34:07 +0300 Subject: [PATCH] + rubynext.y: add method reference operator From https://github.com/whitequark/parser/pull/556 --- lib/parser/lexer.rl | 18 ++++ lib/parser/ruby-next/AST_FORMAT.md | 14 +++ lib/parser/ruby-next/builder.rb | 14 +++ lib/parser/rubynext.y | 6 +- test/ruby-next/test_lexer.rb | 138 +++++++++++++++++++++++++++++ test/ruby-next/test_parser.rb | 70 +++++++++++++++ 6 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 lib/parser/ruby-next/AST_FORMAT.md create mode 100644 lib/parser/ruby-next/builder.rb create mode 100644 test/ruby-next/test_lexer.rb create mode 100644 test/ruby-next/test_parser.rb diff --git a/lib/parser/lexer.rl b/lib/parser/lexer.rl index 74fe2e0b6..d0285b857 100644 --- a/lib/parser/lexer.rl +++ b/lib/parser/lexer.rl @@ -2357,6 +2357,24 @@ class Parser::Lexer # METHOD CALLS # + '.:' w_space+ + => { emit(:tDOT, '.', @ts, @ts + 1) + emit(:tCOLON, ':', @ts + 1, @ts + 2) + p = p - tok.length + 2 + fnext expr_dot; fbreak; }; + + '.:' + => { + if @version >= 27 + emit_table(PUNCTUATION) + else + emit(:tDOT, tok(@ts, @ts + 1), @ts, @ts + 1) + fhold; + end + + fnext expr_dot; fbreak; + }; + '.' | '&.' | '::' => { emit_table(PUNCTUATION) fnext expr_dot; fbreak; }; diff --git a/lib/parser/ruby-next/AST_FORMAT.md b/lib/parser/ruby-next/AST_FORMAT.md new file mode 100644 index 000000000..b097a76ae --- /dev/null +++ b/lib/parser/ruby-next/AST_FORMAT.md @@ -0,0 +1,14 @@ +Ruby Next AST format additions +======================= + +### Method reference operator + +Format: + +~~~ +(meth-ref (self) :foo) +"self.:foo" + ^^ dot + ^^^ selector + ^^^^^^^^^ expression +~~~ diff --git a/lib/parser/ruby-next/builder.rb b/lib/parser/ruby-next/builder.rb new file mode 100644 index 000000000..9119fda98 --- /dev/null +++ b/lib/parser/ruby-next/builder.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "parser/builders/default" +require_relative "meta" + +module Parser + # Add RubyNext specific builder methods + module Builders::Next + def method_ref(receiver, dot_t, selector_t) + n(:meth_ref, [ receiver, value(selector_t).to_sym ], + send_map(receiver, dot_t, selector_t, nil, [], nil)) + end + end +end diff --git a/lib/parser/rubynext.y b/lib/parser/rubynext.y index dd456a51c..dff10806e 100644 --- a/lib/parser/rubynext.y +++ b/lib/parser/rubynext.y @@ -17,7 +17,7 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS tWORDS_BEG tQWORDS_BEG tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DBEG tSTRING_DVAR tSTRING_END tSTRING_DEND tSTRING tSYMBOL tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA tLAMBEG tCHARACTER - tRATIONAL tIMAGINARY tLABEL_END tANDDOT tBDOT2 tBDOT3 + tRATIONAL tIMAGINARY tLABEL_END tANDDOT tMETHREF tBDOT2 tBDOT3 prechigh right tBANG tTILDE tUPLUS @@ -1288,6 +1288,10 @@ rule { result = @builder.keyword_cmd(:retry, val[0]) } + | primary_value tMETHREF operation2 + { + result = @builder.method_ref(val[0], val[1], val[2]) + } primary_value: primary diff --git a/test/ruby-next/test_lexer.rb b/test/ruby-next/test_lexer.rb new file mode 100644 index 000000000..35c0af9b8 --- /dev/null +++ b/test/ruby-next/test_lexer.rb @@ -0,0 +1,138 @@ +# encoding: ascii-8bit +# frozen_string_literal: true + +require 'helper' +require 'complex' + +require 'parser/ruby-next/lexer' + +class TestLexerNext < Minitest::Test + def setup_lexer(version) + @lex = version == "next" ? Parser::Lexer::Next.new(28) : Parser::Lexer.new(version) + + @lex.comments = [] + @lex.diagnostics = Parser::Diagnostic::Engine.new + @lex.diagnostics.all_errors_are_fatal = true + # @lex.diagnostics.consumer = lambda { |diag| $stderr.puts "", diag.render } + end + + def setup + setup_lexer 18 + end + + def utf(str) + str.dup.force_encoding(Encoding::UTF_8) + end + + # + # Additional matchers + # + + def refute_scanned(s, *args) + assert_raises Parser::SyntaxError do + assert_scanned(s, *args) + end + end + + def assert_escape(expected, input) + source_buffer = Parser::Source::Buffer.new('(assert_escape)') + + source_buffer.source = "\"\\#{input}\"".encode(input.encoding) + + @lex.reset + @lex.source_buffer = source_buffer + + lex_token, (lex_value, *) = @lex.advance + + lex_value.force_encoding(Encoding::BINARY) + + assert_equal [:tSTRING, expected], + [lex_token, lex_value], + source_buffer.source + end + + def refute_escape(input) + err = assert_raises Parser::SyntaxError do + @lex.state = :expr_beg + assert_scanned "%Q[\\#{input}]" + end + assert_equal :fatal, err.diagnostic.level + end + + def assert_lex_fname(name, type, range) + begin_pos, end_pos = range + assert_scanned("def #{name} ", + :kDEF, 'def', [0, 3], + type, name, [begin_pos + 4, end_pos + 4]) + + assert_equal :expr_endfn, @lex.state + end + + def assert_scanned(input, *args) + source_buffer = Parser::Source::Buffer.new('(assert_scanned)') + source_buffer.source = input + + @lex.reset(false) + @lex.source_buffer = source_buffer + + until args.empty? do + token, value, (begin_pos, end_pos) = args.shift(3) + + lex_token, (lex_value, lex_range) = @lex.advance + assert lex_token, 'no more tokens' + assert_operator [lex_token, lex_value], :eql?, [token, value], input + assert_equal begin_pos, lex_range.begin_pos + assert_equal end_pos, lex_range.end_pos + end + + lex_token, (lex_value, *) = @lex.advance + refute lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}" + end + + def test_meth_ref + setup_lexer "next" + + assert_scanned('foo.:bar', + :tIDENTIFIER, 'foo', [0, 3], + :tMETHREF, '.:', [3, 5], + :tIDENTIFIER, 'bar', [5, 8]) + + assert_scanned('foo .:bar', + :tIDENTIFIER, 'foo', [0, 3], + :tMETHREF, '.:', [4, 6], + :tIDENTIFIER, 'bar', [6, 9]) + end + + def test_meth_ref_unary_op + setup_lexer "next" + + assert_scanned('foo.:+', + :tIDENTIFIER, 'foo', [0, 3], + :tMETHREF, '.:', [3, 5], + :tPLUS, '+', [5, 6]) + + assert_scanned('foo.:-@', + :tIDENTIFIER, 'foo', [0, 3], + :tMETHREF, '.:', [3, 5], + :tUMINUS, '-@', [5, 7]) + end + + def test_meth_ref_unsupported_newlines + setup_lexer "next" + + # MRI emits exactly the same sequence of tokens, + # the error happens later in the parser + + assert_scanned('foo. :+', + :tIDENTIFIER, 'foo', [0, 3], + :tDOT, '.', [3, 4], + :tCOLON, ':', [5, 6], + :tUPLUS, '+', [6, 7]) + + assert_scanned('foo.: +', + :tIDENTIFIER, 'foo', [0, 3], + :tDOT, '.', [3, 4], + :tCOLON, ':', [4, 5], + :tPLUS, '+', [6, 7]) + end +end diff --git a/test/ruby-next/test_parser.rb b/test/ruby-next/test_parser.rb new file mode 100644 index 000000000..3aea764b9 --- /dev/null +++ b/test/ruby-next/test_parser.rb @@ -0,0 +1,70 @@ +# encoding: utf-8 +# frozen_string_literal: true + +require 'helper' +require 'parse_helper' + +Parser::Builders::Default.modernize + +class TestParser < Minitest::Test + include ParseHelper + + def parser_for_ruby_version(version) + parser = super + parser.diagnostics.all_errors_are_fatal = true + + %w(foo bar baz).each do |metasyntactic_var| + parser.static_env.declare(metasyntactic_var) + end + + parser + end + + SINCE_NEXT = %w(next) + + def test_meth_ref__27 + assert_parses( + s(:meth_ref, s(:lvar, :foo), :bar), + %q{foo.:bar}, + %q{ ^^ dot + | ~~~ selector + |~~~~~~~~ expression}, + SINCE_NEXT) + + assert_parses( + s(:meth_ref, s(:lvar, :foo), :+@), + %q{foo.:+@}, + %q{ ^^ dot + | ~~ selector + |~~~~~~~ expression}, + SINCE_NEXT) + end + + def test_meth_ref__before_27 + assert_diagnoses( + [:error, :unexpected_token, { :token => 'tCOLON' }], + %q{foo.:bar}, + %q{ ^ location }, + ALL_VERSIONS - SINCE_NEXT) + + assert_diagnoses( + [:error, :unexpected_token, { :token => 'tCOLON' }], + %q{foo.:+@}, + %q{ ^ location }, + ALL_VERSIONS - SINCE_NEXT) + end + + def test_meth_ref_unsupported_newlines + assert_diagnoses( + [:error, :unexpected_token, { :token => 'tCOLON' }], + %Q{foo. :+}, + %q{ ^ location}, + SINCE_NEXT) + + assert_diagnoses( + [:error, :unexpected_token, { :token => 'tCOLON' }], + %Q{foo.: +}, + %q{ ^ location}, + SINCE_NEXT) + end +end