diff --git a/config.yml b/config.yml index cc5eb7e099..4892089c03 100644 --- a/config.yml +++ b/config.yml @@ -359,6 +359,8 @@ tokens: comment: "when" - name: NEWLINE comment: "a newline character outside of other tokens" + - name: NEWLINE_TERMINATOR + comment: "a newline that terminates a construct where a newline is otherwise insignificant" - name: PARENTHESIS_RIGHT comment: ")" - name: PIPE diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 749f11173a..a2ad69cd29 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -191,6 +191,7 @@ def deconstruct_keys(keys) # :nodoc: MINUS_EQUAL: :on_op, MINUS_GREATER: :on_tlambda, NEWLINE: :on_nl, + NEWLINE_TERMINATOR: :on_ignored_nl, NUMBERED_REFERENCE: :on_backref, PARENTHESIS_LEFT: :on_lparen, PARENTHESIS_LEFT_GROUPING: :on_lparen, @@ -617,6 +618,9 @@ def result bom = source.slice(0, 3) == "\xEF\xBB\xBF" + last_comment_token = nil #: lex_compat_token? + last_comment_end = nil #: Integer? + result_value.each_with_index do |(prism_token, prism_state), index| lineno = prism_token.location.start_line column = prism_token.location.start_column @@ -625,6 +629,16 @@ def result value = prism_token.value lex_state = Translation::Ripper::Lexer::State[prism_state] + # A comment token does not include its terminating newline, but + # ripper's comment value does, so the newline token that directly + # follows a comment is folded back into it. + if last_comment_token && last_comment_end == prism_token.location.start_offset && (event == :on_nl || event == :on_ignored_nl) + last_comment_token[2] += value + last_comment_token = nil + last_comment_end = nil + next + end + # If there's a UTF-8 byte-order mark as the start of the file, then for # certain tokens ripper sets the first token back by 3 bytes. It also # keeps the byte order mark in the first token's value. This is weird, @@ -714,11 +728,16 @@ def result eof_token = prism_token previous_token = result_value[index - 1][0] + # A newline that was folded back into a comment still marks the + # comment boundary for the check below. + comment_boundary = previous_token.type == :COMMENT || + (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2][0].type == :COMMENT && result_value[index - 2][0].location.end_offset == previous_token.location.start_offset) + # If we're at the end of the file and the previous token was a # comment and there is still whitespace after the comment, then # Ripper will append a on_nl token (even though there isn't # necessarily a newline). We mirror that here. - if previous_token.type == :COMMENT + if comment_boundary # If the comment is at the start of a heredoc: <= 0 - next_token, _ = lexed[index] - - is_inline_comment = prev_token&.location&.start_line == token.location.start_line - if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type) - tokens << [:tCOMMENT, [value, location]] - - nl_location = range(token.location.end_offset - 1, token.location.end_offset) - tokens << [:tNL, [nil, nl_location]] - next - elsif is_inline_comment && next_token&.type == :COMMENT - comment_newline_location = range(token.location.end_offset - 1, token.location.end_offset) - elsif comment_newline_location && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type) - tokens << [:tCOMMENT, [value, location]] - tokens << [:tNL, [nil, comment_newline_location]] - comment_newline_location = nil - next - end + # A carriage return before the terminating newline is part of + # the comment token but not of the comment's value. + location = range(token.location.start_offset, token.location.end_offset - 1) if value.chomp! end when :tNL next_token, _ = lexed[index] @@ -501,6 +486,10 @@ def to_a end end + if comment_newline_location + tokens << [:tNL, [nil, comment_newline_location]] + end + tokens end diff --git a/src/prism.c b/src/prism.c index 566be9e56d..0f9168ad6b 100644 --- a/src/prism.c +++ b/src/prism.c @@ -10212,7 +10212,6 @@ parser_lex(pm_parser_t *parser) { pm_comment_t *comment = parser_comment(parser, PM_COMMENT_INLINE); pm_list_append(&parser->comment_list, (pm_list_node_t *) comment); - if (ending) parser->current.end++; parser->current.type = PM_TOKEN_COMMENT; parser_lex_callback(parser); @@ -10230,7 +10229,16 @@ parser_lex(pm_parser_t *parser) { } } - lexed_comment = true; + /* The comment does not include its terminating newline, + * which lexes through the newline handling below as its + * own token. A comment that ends the file has no newline, + * so the newline handling runs without one to emit. */ + if (ending == NULL) { + lexed_comment = true; + } else { + parser->current.start = ending; + parser->current.end = ending + 1; + } } PRISM_FALLTHROUGH case '\r': @@ -10268,7 +10276,11 @@ parser_lex(pm_parser_t *parser) { break; case PM_IGNORED_NEWLINE_PATTERN: if (parser->pattern_matching_newlines || parser->in_keyword_arg) { - if (!lexed_comment) parser_lex_ignored_newline(parser); + if (!lexed_comment) { + parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR; + parser_lex_callback(parser); + } + lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; parser->current.type = PM_TOKEN_NEWLINE; @@ -10365,11 +10377,15 @@ parser_lex(pm_parser_t *parser) { // If we hit a . after a newline, then we're in a call chain and // we need to return the call operator. if (next_content[0] == '.') { - // To match ripper, we need to emit an ignored newline even though - // it's a real newline in the case that we have a beginless range - // on a subsequent line. + /* A beginless range on the next line means this + * newline terminates the statement rather than + * continuing a method chain. */ if (peek_at(parser, next_content + 1) == '.') { - if (!lexed_comment) parser_lex_ignored_newline(parser); + if (!lexed_comment) { + parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR; + parser_lex_callback(parser); + } + lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; parser->current.type = PM_TOKEN_NEWLINE; diff --git a/templates/src/tokens.c.erb b/templates/src/tokens.c.erb index fb71afe217..6e88d423c2 100644 --- a/templates/src/tokens.c.erb +++ b/templates/src/tokens.c.erb @@ -275,6 +275,8 @@ pm_token_str(pm_token_type_t token_type) { return "'->'"; case PM_TOKEN_NEWLINE: return "newline"; + case PM_TOKEN_NEWLINE_TERMINATOR: + return "newline"; case PM_TOKEN_NUMBERED_REFERENCE: return "numbered reference"; case PM_TOKEN_PARENTHESIS_LEFT: diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb index e44bc20d4d..076f84765c 100644 --- a/test/prism/ruby/parser_test.rb +++ b/test/prism/ruby/parser_test.rb @@ -109,26 +109,13 @@ class ParserTest < TestCase # These files are failing to translate their lexer output into the lexer # output expected by the parser gem, so we'll skip them for now. skip_tokens = [ - "dash_heredocs.txt", "embdoc_no_newline_at_end.txt", - "seattlerb/case_in.txt", - "seattlerb/difficult4__leading_dots2.txt", "seattlerb/heredoc_unicode.txt", "seattlerb/parse_line_heredoc.txt", "seattlerb/pct_w_heredoc_interp_nested.txt", - "seattlerb/required_kwarg_no_value.txt", - "seattlerb/TestRubyParserShared.txt", "unparser/corpus/literal/assignment.txt", "unparser/corpus/literal/literal.txt", - "whitequark/args.txt", - "whitequark/beginless_erange_after_newline.txt", - "whitequark/beginless_irange_after_newline.txt", - "whitequark/forward_arg_with_open_args.txt", - "whitequark/kwarg_no_paren.txt", - "whitequark/multiple_pattern_matches.txt", - "whitequark/newline_in_hash_argument.txt", - "whitequark/pattern_matching_hash.txt", - "whitequark/ruby_bug_9669.txt" + "whitequark/forward_arg_with_open_args.txt" ] Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture|