Split up lbrace token

This also more closely matches parse.y, and makes it easier to
delineate which brace belongs to which type.

Author: kddnewton

config.yml

@@ -388,6 +388,10 @@ tokens:
     comment: "!~"
   - name: BRACE_LEFT
     comment: "{"
+  - name: BRACE_LEFT_ARGUMENT
+    comment: "{ for a block following a parenthesized argument"
+  - name: BRACE_LEFT_HASH
+    comment: "{ for a hash literal"
   - name: BRACKET_LEFT
     comment: "["
   - name: BRACKET_LEFT_ARRAY

lib/prism/lex_compat.rb

@@ -78,6 +78,8 @@ def deconstruct_keys(keys) # :nodoc:
       BANG_EQUAL: :on_op,
       BANG_TILDE: :on_op,
       BRACE_LEFT: :on_lbrace,
+      BRACE_LEFT_ARGUMENT: :on_lbrace,
+      BRACE_LEFT_HASH: :on_lbrace,
       BRACE_RIGHT: :on_rbrace,
       BRACKET_LEFT: :on_lbracket,
       BRACKET_LEFT_ARRAY: :on_lbracket,

lib/prism/translation/parser/lexer.rb

@@ -33,6 +33,8 @@ class Lexer # :nodoc:
           BANG_EQUAL: :tNEQ,
           BANG_TILDE: :tNMATCH,
           BRACE_LEFT: :tLCURLY,
+          BRACE_LEFT_ARGUMENT: :tLBRACE_ARG,
+          BRACE_LEFT_HASH: :tLBRACE,
           BRACE_RIGHT: :tRCURLY,
           BRACKET_LEFT: :tLBRACK2,
           BRACKET_LEFT_ARRAY: :tLBRACK,
@@ -184,16 +186,6 @@ class Lexer # :nodoc:
           WORDS_SEP: :tSPACE
         }
 
-        # These constants represent flags in our lex state. We really, really
-        # don't want to be using them and we really, really don't want to be
-        # exposing them as part of our public API. Unfortunately, we don't have
-        # another way of matching the exact tokens that the parser gem expects
-        # without them. We should find another way to do this, but in the
-        # meantime we'll hide them from the documentation and mark them as
-        # private constants.
-        EXPR_BEG = 0x1
-        EXPR_LABEL = 0x400
-
         # Types of tokens that are allowed to continue a method call with comments in-between.
         # For these, the parser gem doesn't emit a newline token after the last comment.
         COMMENT_CONTINUATION_TYPES = Set.new([:COMMENT, :AMPERSAND_DOT, :DOT])
@@ -202,7 +194,7 @@ class Lexer # :nodoc:
         # Heredocs are complex and require us to keep track of a bit of info to refer to later
         HeredocData = Struct.new(:identifier, :common_whitespace, keyword_init: true)
 
-        private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL, :HeredocData
+        private_constant :TYPES, :HeredocData
 
         # The Parser::Source::Buffer that the tokens were lexed from.
         attr_reader :source_buffer
@@ -241,7 +233,7 @@ def to_a
           comment_newline_location = nil
 
           while index < length
-            token, state = lexed[index]
+            token, _ = lexed[index]
             index += 1
             next if TYPES_ALWAYS_SKIP.include?(token.type)
 
@@ -312,8 +304,6 @@ def to_a
               value.chomp!(":")
             when :tLABEL_END
               value.chomp!(":")
-            when :tLCURLY
-              type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
             when :tNTH_REF
               value = parse_integer(value.delete_prefix("$"))
             when :tOP_ASGN

src/prism.c

@@ -10544,24 +10544,28 @@ parser_lex(pm_parser_t *parser) {
                     pm_token_type_t type = PM_TOKEN_BRACE_LEFT;
 
                     if (parser->enclosure_nesting == parser->lambda_enclosure_nesting) {
-                        // This { begins a lambda
+                        /* This { begins a lambda */
                         parser->command_start = true;
                         lex_state_set(parser, PM_LEX_STATE_BEG);
                         type = PM_TOKEN_LAMBDA_BEGIN;
                     } else if (lex_state_p(parser, PM_LEX_STATE_LABELED)) {
-                        // This { begins a hash literal
+                        /* This { begins a hash literal */
                         lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
+                        type = PM_TOKEN_BRACE_LEFT_HASH;
                     } else if (lex_state_p(parser, PM_LEX_STATE_ARG_ANY | PM_LEX_STATE_END | PM_LEX_STATE_ENDFN)) {
-                        // This { begins a block
+                        /* This { begins a block */
                         parser->command_start = true;
                         lex_state_set(parser, PM_LEX_STATE_BEG);
                     } else if (lex_state_p(parser, PM_LEX_STATE_ENDARG)) {
-                        // This { begins a block on a command
+                        /* This { begins a block following a parenthesized
+                         * command argument */
                         parser->command_start = true;
                         lex_state_set(parser, PM_LEX_STATE_BEG);
+                        type = PM_TOKEN_BRACE_LEFT_ARGUMENT;
                     } else {
-                        // This { begins a hash literal
+                        /* This { begins a hash literal */
                         lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
+                        type = PM_TOKEN_BRACE_LEFT_HASH;
                     }
 
                     parser->enclosure_nesting++;
@@ -13780,7 +13784,7 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod
                 pm_token_t operator = parser->previous;
                 pm_node_t *value = NULL;
 
-                if (match1(parser, PM_TOKEN_BRACE_LEFT)) {
+                if (match1(parser, PM_TOKEN_BRACE_LEFT_HASH)) {
                     // If we're about to parse a nested hash that is being
                     // pushed into this hash directly with **, then we want the
                     // inner hash to share the static literals with the outer
@@ -15398,7 +15402,7 @@ parse_block(pm_parser_t *parser, uint16_t depth) {
      * managed by the lexer. A `do`/`end` block is delimited by keywords, so we
      * push the frame here (covering the block parameters and body) and pop it
      * before consuming `end`, mirroring parse.y's `do_body` rule. */
-    bool do_block = opening.type != PM_TOKEN_BRACE_LEFT;
+    bool do_block = opening.type != PM_TOKEN_BRACE_LEFT && opening.type != PM_TOKEN_BRACE_LEFT_ARGUMENT;
     if (do_block) pm_accepts_block_stack_push(parser, true);
     pm_parser_scope_push(parser, false);
 
@@ -15423,7 +15427,7 @@ parse_block(pm_parser_t *parser, uint16_t depth) {
     accept1(parser, PM_TOKEN_NEWLINE);
     pm_node_t *statements = NULL;
 
-    if (opening.type == PM_TOKEN_BRACE_LEFT) {
+    if (!do_block) {
         if (!match1(parser, PM_TOKEN_BRACE_RIGHT)) {
             statements = UP(parse_statements(parser, PM_CONTEXT_BLOCK_BRACES, (uint16_t) (depth + 1)));
         }
@@ -15542,7 +15546,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a
          * it, pop the command-args frame beneath it, and restore the block
          * frame so the block's `}` still pops it. This mirrors the `tLBRACE_ARG`
          * lookahead handling in parse.y's `command_args` rule. */
-        bool lookahead_brace = match1(parser, PM_TOKEN_BRACE_LEFT);
+        bool lookahead_brace = match2(parser, PM_TOKEN_BRACE_LEFT, PM_TOKEN_BRACE_LEFT_ARGUMENT);
         if (lookahead_brace) pm_accepts_block_stack_pop(parser);
         pm_accepts_block_stack_pop(parser);
         if (lookahead_brace) pm_accepts_block_stack_push(parser, true);
@@ -15554,7 +15558,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a
     if (full_arguments) {
         pm_block_node_t *block = NULL;
 
-        if (accept1(parser, PM_TOKEN_BRACE_LEFT)) {
+        if (accept2(parser, PM_TOKEN_BRACE_LEFT, PM_TOKEN_BRACE_LEFT_ARGUMENT)) {
             found |= true;
             block = parse_block(parser, (uint16_t) (depth + 1));
             pm_arguments_validate_block(parser, arguments, block);
@@ -17359,7 +17363,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm
             pm_array_pattern_node_requireds_append(parser->arena, node, inner);
             return UP(node);
         }
-        case PM_TOKEN_BRACE_LEFT: {
+        case PM_TOKEN_BRACE_LEFT_HASH: {
             bool previous_pattern_matching_newlines = parser->pattern_matching_newlines;
             parser->pattern_matching_newlines = false;
 
@@ -17580,7 +17584,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p
         switch (parser->current.type) {
             case PM_TOKEN_IDENTIFIER:
             case PM_TOKEN_BRACKET_LEFT_ARRAY:
-            case PM_TOKEN_BRACE_LEFT:
+            case PM_TOKEN_BRACE_LEFT_HASH:
             case PM_TOKEN_CARET:
             case PM_TOKEN_CONSTANT:
             case PM_TOKEN_UCOLON_COLON:
@@ -19180,6 +19184,13 @@ parse_parentheses(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t
     /* If this is the end of the file or we match a right parenthesis, then we
      * have an empty parentheses node, and we can immediately return. */
     if (match2(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_EOF)) {
+        /* A command argument group sets EXPR_ENDARG before its ')' is
+         * consumed, even when the group is empty, so that a following '{' is
+         * scanned as a block brace. */
+        if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT) && opening.type == PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES) {
+            lex_state_set(parser, PM_LEX_STATE_ENDARG);
+        }
+
         expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
         pop_block_exits(parser, previous_block_exits);
         return UP(pm_parentheses_node_create(parser, &opening, NULL, &parser->previous, paren_flags));
@@ -19503,7 +19514,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
         case PM_TOKEN_PARENTHESIS_LEFT_GROUPING:
         case PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES:
             return parse_parentheses(parser, binding_power, flags, depth);
-        case PM_TOKEN_BRACE_LEFT: {
+        case PM_TOKEN_BRACE_LEFT_HASH: {
             // If we were passed a current_hash_keys via the parser, then that
             // means we're already parsing a hash and we want to share the set
             // of hash keys with this inner hash we're about to parse for the

templates/src/tokens.c.erb

@@ -53,6 +53,10 @@ pm_token_str(pm_token_type_t token_type) {
             return "'!~'";
         case PM_TOKEN_BRACE_LEFT:
             return "'{'";
+        case PM_TOKEN_BRACE_LEFT_ARGUMENT:
+            return "'{'";
+        case PM_TOKEN_BRACE_LEFT_HASH:
+            return "'{'";
         case PM_TOKEN_BRACE_RIGHT:
             return "'}'";
         case PM_TOKEN_BRACKET_LEFT:

test/prism/errors/command_calls_25.txt

@@ -3,6 +3,7 @@
       ^ expected a `do` keyword or a `{` to open the lambda block
         ^ unexpected ')', expecting end-of-input
         ^ unexpected ')', ignoring it
-           ^ unexpected end-of-input, assuming it is closing the parent top level context
+         ^ unexpected '{', ignoring it
+          ^ unexpected '}', ignoring it
 ^~ expected a lambda block beginning with `do` to end with `end`
 

test/prism/ruby/parser_test.rb

@@ -111,11 +111,8 @@ class ParserTest < TestCase
     skip_tokens = [
       "dash_heredocs.txt",
       "embdoc_no_newline_at_end.txt",
-      "seattlerb/bug169.txt",
       "seattlerb/case_in.txt",
       "seattlerb/difficult4__leading_dots2.txt",
-      "seattlerb/difficult6__7.txt",
-      "seattlerb/difficult6__8.txt",
       "seattlerb/heredoc_unicode.txt",
       "seattlerb/parse_line_heredoc.txt",
       "seattlerb/pct_w_heredoc_interp_nested.txt",
@@ -128,14 +125,10 @@ class ParserTest < TestCase
       "whitequark/beginless_irange_after_newline.txt",
       "whitequark/forward_arg_with_open_args.txt",
       "whitequark/kwarg_no_paren.txt",
-      "whitequark/lbrace_arg_after_command_args.txt",
       "whitequark/multiple_pattern_matches.txt",
       "whitequark/newline_in_hash_argument.txt",
       "whitequark/pattern_matching_hash.txt",
-      "whitequark/ruby_bug_14690.txt",
-      "whitequark/ruby_bug_9669.txt",
-      "whitequark/space_args_arg_block.txt",
-      "whitequark/space_args_block.txt"
+      "whitequark/ruby_bug_9669.txt"
     ]
 
     Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture|