+ ruby27.y: Added numbered parameters support. (#565)

This commit tracks upstream commits
  - ruby/ruby@12acc75
  - ruby/ruby@d0e25ed
  - ruby/ruby@964bbc1
  - ruby/ruby@ae07b66
  - ruby/ruby@6ca9e7c
  - ruby/ruby@b8f3be2
  - ruby/ruby@bb4ac7a

doc/AST_FORMAT.md

@@ -1083,6 +1083,40 @@ However, the following code results in a parse error:
 def f(*a: b); end
 ~~~
 
+## Numbered parameters
+
+### Block with numbered parameters
+
+Ruby 2.7 introduced a feature called "numbered parameters".
+Numbered and ordinal parameters are mutually exclusive, so if the block
+has only numbered parameters it also has a different AST node.
+
+Note that the second child represents a total number of numbered parameters.
+
+Format:
+
+~~~
+s(:numblock,
+  s(:send, nil, :proc), 3,
+  s(:send,
+    s(:numparam, 1), :+,
+    s(:numparam, 3)))
+"proc { @1 + @3 }"
+      ~ begin   ~ end
+ ~~~~~~~~~~~~~~~~ expression
+~~~
+
+### Numbered parameter
+
+Format:
+
+~~~
+(numparam 10)
+"@10"
+ ~~~ name
+ ~~~ expression
+~~~
+
 ## Send
 
 ### To self

lib/parser.rb

@@ -67,6 +67,7 @@ module Source
   require 'parser/lexer/literal'
   require 'parser/lexer/stack_state'
   require 'parser/lexer/dedenter'
+  require 'parser/lexer/max_numparam_stack'
 
   module Builders
     require 'parser/builders/default'

lib/parser/ast/processor.rb

@@ -173,6 +173,14 @@ def on_send(node)
       alias on_block    process_regular_node
       alias on_lambda   process_regular_node
 
+      def on_numblock(node)
+        method_call, max_numparam, body = *node
+
+        node.updated(nil, [
+          process(method_call), max_numparam, process(body)
+        ])
+      end
+
       alias on_while      process_regular_node
       alias on_while_post process_regular_node
       alias on_until      process_regular_node

lib/parser/builders/default.rb

@@ -437,6 +437,11 @@ def cvar(token)
         variable_map(token))
     end
 
+    def numparam(token)
+      n(:numparam, [ value(token).to_i ],
+        variable_map(token))
+    end
+
     def back_ref(token)
       n(:back_ref, [ value(token).to_sym ],
         token_map(token))
@@ -663,6 +668,10 @@ def args(begin_t, args, end_t, check_args=true)
         collection_map(begin_t, args, end_t))
     end
 
+    def numargs(max_numparam)
+      n(:numargs, [ max_numparam ], nil)
+    end
+
     def arg(name_t)
       n(:arg, [ value(name_t).to_sym ],
         variable_map(name_t))
@@ -835,15 +844,23 @@ def block(method_call, begin_t, args, body, end_t)
         diagnostic :error, :block_and_blockarg, nil, last_arg.loc.expression, [loc(begin_t)]
       end
 
+
+      if args.type == :numargs
+        block_type = :numblock
+        args = args.children[0]
+      else
+        block_type = :block
+      end
+
       if [:send, :csend, :index, :super, :zsuper, :lambda].include?(method_call.type)
-        n(:block, [ method_call, args, body ],
+        n(block_type, [ method_call, args, body ],
           block_map(method_call.loc.expression, begin_t, end_t))
       else
         # Code like "return foo 1 do end" is reduced in a weird sequence.
         # Here, method_call is actually (return).
         actual_send, = *method_call
         block =
-          n(:block, [ actual_send, args, body ],
+          n(block_type, [ actual_send, args, body ],
             block_map(actual_send.loc.expression, begin_t, end_t))
 
         n(method_call.type, [ block ],

lib/parser/context.rb

@@ -47,5 +47,13 @@ def class_definition_allowed?
     end
     alias module_definition_allowed? class_definition_allowed?
     alias dynamic_const_definition_allowed? class_definition_allowed?
+
+    def in_block?
+      @stack.last == :block
+    end
+
+    def in_lambda?
+      @stack.last == :lambda
+    end
   end
 end

lib/parser/lexer.rl

@@ -89,7 +89,10 @@ class Parser::Lexer
 
   REGEXP_META_CHARACTERS = Regexp.union(*"\\$()*+.<>?[]^{|}".chars).freeze
 
+  NUMPARAM_MAX = 100
+
   attr_reader   :source_buffer
+  attr_reader   :max_numparam_stack
 
   attr_accessor :diagnostics
   attr_accessor :static_env
@@ -176,6 +179,9 @@ class Parser::Lexer
 
     # State before =begin / =end block comment
     @cs_before_block_comment = self.class.lex_en_line_begin
+
+    # Maximum numbered parameters stack
+    @max_numparam_stack = MaxNumparamStack.new
   end
 
   def source_buffer=(source_buffer)
@@ -249,6 +255,10 @@ class Parser::Lexer
     @cond = @cond_stack.pop
   end
 
+  def max_numparam
+    @max_numparam_stack.top
+  end
+
   def dedent_level
     # We erase @dedent_level as a precaution to avoid accidentally
     # using a stale value.
@@ -1301,6 +1311,36 @@ class Parser::Lexer
         fnext *stack_pop; fbreak;
       };
 
+      '@' [0-9]+
+      => {
+        if @version < 27
+          diagnostic :error, :ivar_name, { :name => tok }
+        end
+
+        value = tok[1..-1]
+
+        if value[0] == '0'
+          diagnostic :error, :leading_zero_in_numparam, nil, range(@ts, @te)
+        end
+
+        if value.to_i > NUMPARAM_MAX
+          diagnostic :error, :too_large_numparam, nil, range(@ts, @te)
+        end
+
+        if !@context.in_block? && !@context.in_lambda?
+          diagnostic :error, :numparam_outside_block, nil, range(@ts, @te)
+        end
+
+        if !@max_numparam_stack.can_have_numparams?
+          diagnostic :error, :ordinary_param_defined, nil, range(@ts, @te)
+        end
+
+        @max_numparam_stack.register(value.to_i)
+
+        emit(:tNUMPARAM, tok[1..-1])
+        fnext *stack_pop; fbreak;
+      };
+
       instance_var_v
       => {
         if tok =~ /^@[0-9]/

lib/parser/lexer/max_numparam_stack.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Parser
+
+  class Lexer::MaxNumparamStack
+    def initialize
+      @stack = []
+    end
+
+    def cant_have_numparams!
+      set(-1)
+    end
+
+    def can_have_numparams?
+      top >= 0
+    end
+
+    def register(numparam)
+      set( [top, numparam].max )
+    end
+
+    def top
+      @stack.last
+    end
+
+    def push
+      @stack.push(0)
+    end
+
+    def pop
+      @stack.pop
+    end
+
+    private
+
+    def set(value)
+      @stack.pop
+      @stack.push(value)
+    end
+  end
+
+end

lib/parser/messages.rb

@@ -10,27 +10,31 @@ module Parser
   #
   MESSAGES = {
     # Lexer errors
-    :unicode_point_too_large => 'invalid Unicode codepoint (too large)',
-    :invalid_escape          => 'invalid escape character syntax',
-    :incomplete_escape       => 'incomplete character syntax',
-    :invalid_hex_escape      => 'invalid hex escape',
-    :invalid_unicode_escape  => 'invalid Unicode escape',
-    :unterminated_unicode    => 'unterminated Unicode escape',
-    :escape_eof              => 'escape sequence meets end of file',
-    :string_eof              => 'unterminated string meets end of file',
-    :regexp_options          => 'unknown regexp options: %{options}',
-    :cvar_name               => "`%{name}' is not allowed as a class variable name",
-    :ivar_name               => "`%{name}' is not allowed as an instance variable name",
-    :trailing_in_number      => "trailing `%{character}' in number",
-    :empty_numeric           => 'numeric literal without digits',
-    :invalid_octal           => 'invalid octal digit',
-    :no_dot_digit_literal    => 'no .<digit> floating literal anymore; put 0 before dot',
-    :bare_backslash          => 'bare backslash only allowed before newline',
-    :unexpected              => "unexpected `%{character}'",
-    :embedded_document       => 'embedded document meets end of file (and they embark on a romantic journey)',
-    :heredoc_id_has_newline  => 'here document identifier across newlines, never match',
-    :heredoc_id_ends_with_nl => 'here document identifier ends with a newline',
-    :unterminated_heredoc_id => 'unterminated heredoc id',
+    :unicode_point_too_large  => 'invalid Unicode codepoint (too large)',
+    :invalid_escape           => 'invalid escape character syntax',
+    :incomplete_escape        => 'incomplete character syntax',
+    :invalid_hex_escape       => 'invalid hex escape',
+    :invalid_unicode_escape   => 'invalid Unicode escape',
+    :unterminated_unicode     => 'unterminated Unicode escape',
+    :escape_eof               => 'escape sequence meets end of file',
+    :string_eof               => 'unterminated string meets end of file',
+    :regexp_options           => 'unknown regexp options: %{options}',
+    :cvar_name                => "`%{name}' is not allowed as a class variable name",
+    :ivar_name                => "`%{name}' is not allowed as an instance variable name",
+    :trailing_in_number       => "trailing `%{character}' in number",
+    :empty_numeric            => 'numeric literal without digits',
+    :invalid_octal            => 'invalid octal digit',
+    :no_dot_digit_literal     => 'no .<digit> floating literal anymore; put 0 before dot',
+    :bare_backslash           => 'bare backslash only allowed before newline',
+    :unexpected               => "unexpected `%{character}'",
+    :embedded_document        => 'embedded document meets end of file (and they embark on a romantic journey)',
+    :heredoc_id_has_newline   => 'here document identifier across newlines, never match',
+    :heredoc_id_ends_with_nl  => 'here document identifier ends with a newline',
+    :unterminated_heredoc_id  => 'unterminated heredoc id',
+    :leading_zero_in_numparam => 'leading zero is not allowed as a numbered parameter',
+    :numparam_outside_block   => 'numbered parameter outside block',
+    :too_large_numparam       => 'too large numbered parameter',
+    :ordinary_param_defined   => 'ordinary parameter is defined',
 
     # Lexer warnings
     :invalid_escape_use      => 'invalid character syntax; use ?%{escape}',

lib/parser/meta.rb

@@ -26,6 +26,7 @@ module class sclass def defs undef alias args
         ident root lambda indexasgn index procarg0
         meth_ref restarg_expr blockarg_expr
         objc_kwarg objc_restarg objc_varargs
+        numargs numblock numparam
       ).map(&:to_sym).to_set.freeze
 
   end # Meta