Split up newline token

There are actually 3 tokens in the Ruby grammar: the newline in a
whitespace insensitive position that is ignored, a newline that is
in a whitespace sensitive position that acts as a statement
terminator, and a newline in a whitespace sensisitive position that
acts as the terminator for an expression. The third one doesn't
exist in our grammar at the moment, but I want to add it, because
it makes working with translating the lex output easier.

Author: kddnewton

config.yml

@@ -359,6 +359,8 @@ tokens:
     comment: "when"
   - name: NEWLINE
     comment: "a newline character outside of other tokens"
+  - name: NEWLINE_TERMINATOR
+    comment: "a newline that terminates a construct where a newline is otherwise insignificant"
   - name: PARENTHESIS_RIGHT
     comment: ")"
   - name: PIPE

lib/prism/lex_compat.rb

@@ -191,6 +191,7 @@ def deconstruct_keys(keys) # :nodoc:
       MINUS_EQUAL: :on_op,
       MINUS_GREATER: :on_tlambda,
       NEWLINE: :on_nl,
+      NEWLINE_TERMINATOR: :on_ignored_nl,
       NUMBERED_REFERENCE: :on_backref,
       PARENTHESIS_LEFT: :on_lparen,
       PARENTHESIS_LEFT_GROUPING: :on_lparen,
@@ -617,6 +618,9 @@ def result
 
       bom = source.slice(0, 3) == "\xEF\xBB\xBF"
 
+      last_comment_token = nil #: lex_compat_token?
+      last_comment_end = nil #: Integer?
+
       result_value.each_with_index do |(prism_token, prism_state), index|
         lineno = prism_token.location.start_line
         column = prism_token.location.start_column
@@ -625,6 +629,16 @@ def result
         value = prism_token.value
         lex_state = Translation::Ripper::Lexer::State[prism_state]
 
+        # A comment token does not include its terminating newline, but
+        # ripper's comment value does, so the newline token that directly
+        # follows a comment is folded back into it.
+        if last_comment_token && last_comment_end == prism_token.location.start_offset && (event == :on_nl || event == :on_ignored_nl)
+          last_comment_token[2] += value
+          last_comment_token = nil
+          last_comment_end = nil
+          next
+        end
+
         # If there's a UTF-8 byte-order mark as the start of the file, then for
         # certain tokens ripper sets the first token back by 3 bytes. It also
         # keeps the byte order mark in the first token's value. This is weird,
@@ -714,11 +728,16 @@ def result
             eof_token = prism_token
             previous_token = result_value[index - 1][0]
 
+            # A newline that was folded back into a comment still marks the
+            # comment boundary for the check below.
+            comment_boundary = previous_token.type == :COMMENT ||
+              (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2][0].type == :COMMENT && result_value[index - 2][0].location.end_offset == previous_token.location.start_offset)
+
             # If we're at the end of the file and the previous token was a
             # comment and there is still whitespace after the comment, then
             # Ripper will append a on_nl token (even though there isn't
             # necessarily a newline). We mirror that here.
-            if previous_token.type == :COMMENT
+            if comment_boundary
               # If the comment is at the start of a heredoc: <<HEREDOC # comment
               # then the comment's end_offset is up near the heredoc_beg.
               # This is not the correct offset to use for figuring out if
@@ -745,6 +764,11 @@ def result
 
         previous_state = lex_state
 
+        if event == :on_comment
+          last_comment_token = lex_compat_token
+          last_comment_end = prism_token.location.end_offset
+        end
+
         # The order in which tokens appear in our lexer is different from the
         # order that they appear in Ripper. When we hit the declaration of a
         # heredoc in prism, we skip forward and lex the rest of the content of

lib/prism/translation/parser/lexer.rb

@@ -141,6 +141,7 @@ class Lexer # :nodoc:
           MINUS_EQUAL: :tOP_ASGN,
           MINUS_GREATER: :tLAMBDA,
           NEWLINE: :tNL,
+          NEWLINE_TERMINATOR: :tNL,
           NUMBERED_REFERENCE: :tNTH_REF,
           PARENTHESIS_LEFT: :tLPAREN2,
           PARENTHESIS_LEFT_GROUPING: :tLPAREN,
@@ -187,11 +188,6 @@ class Lexer # :nodoc:
           XSTRING_BEGIN: :tXSTRING_BEG
         }
 
-        # Types of tokens that are allowed to continue a method call with comments in-between.
-        # For these, the parser gem doesn't emit a newline token after the last comment.
-        COMMENT_CONTINUATION_TYPES = Set.new([:COMMENT, :AMPERSAND_DOT, :DOT])
-        private_constant :COMMENT_CONTINUATION_TYPES
-
         # Heredocs are complex and require us to keep track of a bit of info to refer to later
         HeredocData = Struct.new(:identifier, :common_whitespace, keyword_init: true)
 
@@ -242,6 +238,13 @@ def to_a
             value = token.value
             location = range(token.location.start_offset, token.location.end_offset)
 
+            # A newline deferred past a run of comments is emitted before the
+            # token that follows the last comment.
+            if comment_newline_location && type != :tCOMMENT
+              tokens << [:tNL, [nil, comment_newline_location]]
+              comment_newline_location = nil
+            end
+
             case type
             when :tCHARACTER
               value.delete_prefix!("?")
@@ -259,27 +262,9 @@ def to_a
                 location = range(token.location.start_offset, next_token.location.end_offset)
                 index += 1
               else
-                is_at_eol = value.chomp!.nil?
-                location = range(token.location.start_offset, token.location.end_offset + (is_at_eol ? 0 : -1))
-
-                prev_token, _ = lexed[index - 2] if index - 2 >= 0
-                next_token, _ = lexed[index]
-
-                is_inline_comment = prev_token&.location&.start_line == token.location.start_line
-                if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
-                  tokens << [:tCOMMENT, [value, location]]
-
-                  nl_location = range(token.location.end_offset - 1, token.location.end_offset)
-                  tokens << [:tNL, [nil, nl_location]]
-                  next
-                elsif is_inline_comment && next_token&.type == :COMMENT
-                  comment_newline_location = range(token.location.end_offset - 1, token.location.end_offset)
-                elsif comment_newline_location && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
-                  tokens << [:tCOMMENT, [value, location]]
-                  tokens << [:tNL, [nil, comment_newline_location]]
-                  comment_newline_location = nil
-                  next
-                end
+                # A carriage return before the terminating newline is part of
+                # the comment token but not of the comment's value.
+                location = range(token.location.start_offset, token.location.end_offset - 1) if value.chomp!
               end
             when :tNL
               next_token, _ = lexed[index]
@@ -501,6 +486,10 @@ def to_a
             end
           end
 
+          if comment_newline_location
+            tokens << [:tNL, [nil, comment_newline_location]]
+          end
+
           tokens
         end
 

src/prism.c

@@ -10212,7 +10212,6 @@ parser_lex(pm_parser_t *parser) {
                     pm_comment_t *comment = parser_comment(parser, PM_COMMENT_INLINE);
                     pm_list_append(&parser->comment_list, (pm_list_node_t *) comment);
 
-                    if (ending) parser->current.end++;
                     parser->current.type = PM_TOKEN_COMMENT;
                     parser_lex_callback(parser);
 
@@ -10230,7 +10229,16 @@ parser_lex(pm_parser_t *parser) {
                         }
                     }
 
-                    lexed_comment = true;
+                    /* The comment does not include its terminating newline,
+                     * which lexes through the newline handling below as its
+                     * own token. A comment that ends the file has no newline,
+                     * so the newline handling runs without one to emit. */
+                    if (ending == NULL) {
+                        lexed_comment = true;
+                    } else {
+                        parser->current.start = ending;
+                        parser->current.end = ending + 1;
+                    }
                 }
                 PRISM_FALLTHROUGH
                 case '\r':
@@ -10268,7 +10276,11 @@ parser_lex(pm_parser_t *parser) {
                             break;
                         case PM_IGNORED_NEWLINE_PATTERN:
                             if (parser->pattern_matching_newlines || parser->in_keyword_arg) {
-                                if (!lexed_comment) parser_lex_ignored_newline(parser);
+                                if (!lexed_comment) {
+                                    parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR;
+                                    parser_lex_callback(parser);
+                                }
+
                                 lex_state_set(parser, PM_LEX_STATE_BEG);
                                 parser->command_start = true;
                                 parser->current.type = PM_TOKEN_NEWLINE;
@@ -10365,11 +10377,15 @@ parser_lex(pm_parser_t *parser) {
                         // If we hit a . after a newline, then we're in a call chain and
                         // we need to return the call operator.
                         if (next_content[0] == '.') {
-                            // To match ripper, we need to emit an ignored newline even though
-                            // it's a real newline in the case that we have a beginless range
-                            // on a subsequent line.
+                            /* A beginless range on the next line means this
+                             * newline terminates the statement rather than
+                             * continuing a method chain. */
                             if (peek_at(parser, next_content + 1) == '.') {
-                                if (!lexed_comment) parser_lex_ignored_newline(parser);
+                                if (!lexed_comment) {
+                                    parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR;
+                                    parser_lex_callback(parser);
+                                }
+
                                 lex_state_set(parser, PM_LEX_STATE_BEG);
                                 parser->command_start = true;
                                 parser->current.type = PM_TOKEN_NEWLINE;

templates/src/tokens.c.erb

@@ -275,6 +275,8 @@ pm_token_str(pm_token_type_t token_type) {
             return "'->'";
         case PM_TOKEN_NEWLINE:
             return "newline";
+        case PM_TOKEN_NEWLINE_TERMINATOR:
+            return "newline";
         case PM_TOKEN_NUMBERED_REFERENCE:
             return "numbered reference";
         case PM_TOKEN_PARENTHESIS_LEFT:

test/prism/ruby/parser_test.rb

@@ -109,26 +109,13 @@ class ParserTest < TestCase
     # These files are failing to translate their lexer output into the lexer
     # output expected by the parser gem, so we'll skip them for now.
     skip_tokens = [
-      "dash_heredocs.txt",
       "embdoc_no_newline_at_end.txt",
-      "seattlerb/case_in.txt",
-      "seattlerb/difficult4__leading_dots2.txt",
       "seattlerb/heredoc_unicode.txt",
       "seattlerb/parse_line_heredoc.txt",
       "seattlerb/pct_w_heredoc_interp_nested.txt",
-      "seattlerb/required_kwarg_no_value.txt",
-      "seattlerb/TestRubyParserShared.txt",
       "unparser/corpus/literal/assignment.txt",
       "unparser/corpus/literal/literal.txt",
-      "whitequark/args.txt",
-      "whitequark/beginless_erange_after_newline.txt",
-      "whitequark/beginless_irange_after_newline.txt",
-      "whitequark/forward_arg_with_open_args.txt",
-      "whitequark/kwarg_no_paren.txt",
-      "whitequark/multiple_pattern_matches.txt",
-      "whitequark/newline_in_hash_argument.txt",
-      "whitequark/pattern_matching_hash.txt",
-      "whitequark/ruby_bug_9669.txt"
+      "whitequark/forward_arg_with_open_args.txt"
     ]
 
     Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture|