Revert "Split up newline token"

config.yml

@@ -359,8 +359,6 @@ tokens:
     comment: "when"
   - name: NEWLINE
     comment: "a newline character outside of other tokens"
-  - name: NEWLINE_TERMINATOR
-    comment: "a newline that terminates a construct where a newline is otherwise insignificant"
   - name: PARENTHESIS_RIGHT
     comment: ")"
   - name: PIPE

lib/prism/lex_compat.rb

@@ -191,7 +191,6 @@ def deconstruct_keys(keys) # :nodoc:
       MINUS_EQUAL: :on_op,
       MINUS_GREATER: :on_tlambda,
       NEWLINE: :on_nl,
-      NEWLINE_TERMINATOR: :on_ignored_nl,
       NUMBERED_REFERENCE: :on_backref,
       PARENTHESIS_LEFT: :on_lparen,
       PARENTHESIS_LEFT_GROUPING: :on_lparen,
@@ -618,9 +617,6 @@ def result
 
       bom = source.slice(0, 3) == "\xEF\xBB\xBF"
 
-      last_comment_token = nil #: lex_compat_token?
-      last_comment_end = nil #: Integer?
-
       result_value.each_with_index do |(prism_token, prism_state), index|
         lineno = prism_token.location.start_line
         column = prism_token.location.start_column
@@ -629,16 +625,6 @@ def result
         value = prism_token.value
         lex_state = Translation::Ripper::Lexer::State[prism_state]
 
-        # A comment token does not include its terminating newline, but
-        # ripper's comment value does, so the newline token that directly
-        # follows a comment is folded back into it.
-        if last_comment_token && last_comment_end == prism_token.location.start_offset && (event == :on_nl || event == :on_ignored_nl)
-          last_comment_token[2] += value
-          last_comment_token = nil
-          last_comment_end = nil
-          next
-        end
-
         # If there's a UTF-8 byte-order mark as the start of the file, then for
         # certain tokens ripper sets the first token back by 3 bytes. It also
         # keeps the byte order mark in the first token's value. This is weird,
@@ -728,16 +714,11 @@ def result
             eof_token = prism_token
             previous_token = result_value[index - 1][0]
 
-            # A newline that was folded back into a comment still marks the
-            # comment boundary for the check below.
-            comment_boundary = previous_token.type == :COMMENT ||
-              (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2][0].type == :COMMENT && result_value[index - 2][0].location.end_offset == previous_token.location.start_offset)
-
             # If we're at the end of the file and the previous token was a
             # comment and there is still whitespace after the comment, then
             # Ripper will append a on_nl token (even though there isn't
             # necessarily a newline). We mirror that here.
-            if comment_boundary
+            if previous_token.type == :COMMENT
               # If the comment is at the start of a heredoc: <<HEREDOC # comment
               # then the comment's end_offset is up near the heredoc_beg.
               # This is not the correct offset to use for figuring out if
@@ -764,11 +745,6 @@ def result
 
         previous_state = lex_state
 
-        if event == :on_comment
-          last_comment_token = lex_compat_token
-          last_comment_end = prism_token.location.end_offset
-        end
-
         # The order in which tokens appear in our lexer is different from the
         # order that they appear in Ripper. When we hit the declaration of a
         # heredoc in prism, we skip forward and lex the rest of the content of

lib/prism/translation/parser/lexer.rb

@@ -141,7 +141,6 @@ class Lexer # :nodoc:
           MINUS_EQUAL: :tOP_ASGN,
           MINUS_GREATER: :tLAMBDA,
           NEWLINE: :tNL,
-          NEWLINE_TERMINATOR: :tNL,
           NUMBERED_REFERENCE: :tNTH_REF,
           PARENTHESIS_LEFT: :tLPAREN2,
           PARENTHESIS_LEFT_GROUPING: :tLPAREN,
@@ -188,6 +187,11 @@ class Lexer # :nodoc:
           XSTRING_BEGIN: :tXSTRING_BEG
         }
 
+        # Types of tokens that are allowed to continue a method call with comments in-between.
+        # For these, the parser gem doesn't emit a newline token after the last comment.
+        COMMENT_CONTINUATION_TYPES = Set.new([:COMMENT, :AMPERSAND_DOT, :DOT])
+        private_constant :COMMENT_CONTINUATION_TYPES
+
         # Heredocs are complex and require us to keep track of a bit of info to refer to later
         HeredocData = Struct.new(:identifier, :common_whitespace, keyword_init: true)
 
@@ -238,13 +242,6 @@ def to_a
             value = token.value
             location = range(token.location.start_offset, token.location.end_offset)
 
-            # A newline deferred past a run of comments is emitted before the
-            # token that follows the last comment.
-            if comment_newline_location && type != :tCOMMENT
-              tokens << [:tNL, [nil, comment_newline_location]]
-              comment_newline_location = nil
-            end
-
             case type
             when :tCHARACTER
               value.delete_prefix!("?")
@@ -262,9 +259,27 @@ def to_a
                 location = range(token.location.start_offset, next_token.location.end_offset)
                 index += 1
               else
-                # A carriage return before the terminating newline is part of
-                # the comment token but not of the comment's value.
-                location = range(token.location.start_offset, token.location.end_offset - 1) if value.chomp!
+                is_at_eol = value.chomp!.nil?
+                location = range(token.location.start_offset, token.location.end_offset + (is_at_eol ? 0 : -1))
+
+                prev_token, _ = lexed[index - 2] if index - 2 >= 0
+                next_token, _ = lexed[index]
+
+                is_inline_comment = prev_token&.location&.start_line == token.location.start_line
+                if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
+                  tokens << [:tCOMMENT, [value, location]]
+
+                  nl_location = range(token.location.end_offset - 1, token.location.end_offset)
+                  tokens << [:tNL, [nil, nl_location]]
+                  next
+                elsif is_inline_comment && next_token&.type == :COMMENT
+                  comment_newline_location = range(token.location.end_offset - 1, token.location.end_offset)
+                elsif comment_newline_location && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
+                  tokens << [:tCOMMENT, [value, location]]
+                  tokens << [:tNL, [nil, comment_newline_location]]
+                  comment_newline_location = nil
+                  next
+                end
               end
             when :tNL
               next_token, _ = lexed[index]
@@ -486,10 +501,6 @@ def to_a
             end
           end
 
-          if comment_newline_location
-            tokens << [:tNL, [nil, comment_newline_location]]
-          end
-
           tokens
         end
 

src/prism.c

@@ -10212,6 +10212,7 @@ parser_lex(pm_parser_t *parser) {
                     pm_comment_t *comment = parser_comment(parser, PM_COMMENT_INLINE);
                     pm_list_append(&parser->comment_list, (pm_list_node_t *) comment);
 
+                    if (ending) parser->current.end++;
                     parser->current.type = PM_TOKEN_COMMENT;
                     parser_lex_callback(parser);
 
@@ -10229,16 +10230,7 @@ parser_lex(pm_parser_t *parser) {
                         }
                     }
 
-                    /* The comment does not include its terminating newline,
-                     * which lexes through the newline handling below as its
-                     * own token. A comment that ends the file has no newline,
-                     * so the newline handling runs without one to emit. */
-                    if (ending == NULL) {
-                        lexed_comment = true;
-                    } else {
-                        parser->current.start = ending;
-                        parser->current.end = ending + 1;
-                    }
+                    lexed_comment = true;
                 }
                 PRISM_FALLTHROUGH
                 case '\r':
@@ -10276,11 +10268,7 @@ parser_lex(pm_parser_t *parser) {
                             break;
                         case PM_IGNORED_NEWLINE_PATTERN:
                             if (parser->pattern_matching_newlines || parser->in_keyword_arg) {
-                                if (!lexed_comment) {
-                                    parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR;
-                                    parser_lex_callback(parser);
-                                }
-
+                                if (!lexed_comment) parser_lex_ignored_newline(parser);
                                 lex_state_set(parser, PM_LEX_STATE_BEG);
                                 parser->command_start = true;
                                 parser->current.type = PM_TOKEN_NEWLINE;
@@ -10377,15 +10365,11 @@ parser_lex(pm_parser_t *parser) {
                         // If we hit a . after a newline, then we're in a call chain and
                         // we need to return the call operator.
                         if (next_content[0] == '.') {
-                            /* A beginless range on the next line means this
-                             * newline terminates the statement rather than
-                             * continuing a method chain. */
+                            // To match ripper, we need to emit an ignored newline even though
+                            // it's a real newline in the case that we have a beginless range
+                            // on a subsequent line.
                             if (peek_at(parser, next_content + 1) == '.') {
-                                if (!lexed_comment) {
-                                    parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR;
-                                    parser_lex_callback(parser);
-                                }
-
+                                if (!lexed_comment) parser_lex_ignored_newline(parser);
                                 lex_state_set(parser, PM_LEX_STATE_BEG);
                                 parser->command_start = true;
                                 parser->current.type = PM_TOKEN_NEWLINE;

templates/src/tokens.c.erb

@@ -275,8 +275,6 @@ pm_token_str(pm_token_type_t token_type) {
             return "'->'";
         case PM_TOKEN_NEWLINE:
             return "newline";
-        case PM_TOKEN_NEWLINE_TERMINATOR:
-            return "newline";
         case PM_TOKEN_NUMBERED_REFERENCE:
             return "numbered reference";
         case PM_TOKEN_PARENTHESIS_LEFT:

test/prism/ruby/parser_test.rb

@@ -109,13 +109,26 @@ class ParserTest < TestCase
     # These files are failing to translate their lexer output into the lexer
     # output expected by the parser gem, so we'll skip them for now.
     skip_tokens = [
+      "dash_heredocs.txt",
       "embdoc_no_newline_at_end.txt",
+      "seattlerb/case_in.txt",
+      "seattlerb/difficult4__leading_dots2.txt",
       "seattlerb/heredoc_unicode.txt",
       "seattlerb/parse_line_heredoc.txt",
       "seattlerb/pct_w_heredoc_interp_nested.txt",
+      "seattlerb/required_kwarg_no_value.txt",
+      "seattlerb/TestRubyParserShared.txt",
       "unparser/corpus/literal/assignment.txt",
       "unparser/corpus/literal/literal.txt",
-      "whitequark/forward_arg_with_open_args.txt"
+      "whitequark/args.txt",
+      "whitequark/beginless_erange_after_newline.txt",
+      "whitequark/beginless_irange_after_newline.txt",
+      "whitequark/forward_arg_with_open_args.txt",
+      "whitequark/kwarg_no_paren.txt",
+      "whitequark/multiple_pattern_matches.txt",
+      "whitequark/newline_in_hash_argument.txt",
+      "whitequark/pattern_matching_hash.txt",
+      "whitequark/ruby_bug_9669.txt"
     ]
 
     Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture|