Skip to content

Commit

Permalink
[ruby/prism] Fix parsing heredoc ends
Browse files Browse the repository at this point in the history 
ruby/prism@aa8c702271
  • Loading branch information
@haldun @matzbot
haldun authored and matzbot committed Dec 1, 2023
1 parent 3923888 commit 562d949
Show file tree
Hide file tree
Showing 5 changed files with 147 additions and 28 deletions.
95 changes: 67 additions & 28 deletions prism/prism.c
View file
Expand Up @@ -9761,24 +9761,43 @@ parser_lex(pm_parser_t *parser) {
// terminator, then we need to return the ending of the heredoc.
if (current_token_starts_line(parser)) {
const uint8_t *start = parser->current.start;
size_t whitespace = pm_heredoc_strspn_inline_whitespace(parser, &start, lex_mode->as.heredoc.indent);

if ((start + ident_length <= parser->end) && (memcmp(start, ident_start, ident_length) == 0)) {
bool matched = true;
if (start + ident_length <= parser->end) {
bool at_end = false;
const uint8_t *newline = next_newline(start, parser->end - start);
const uint8_t *ident_end = newline;
const uint8_t *terminator_end = newline;

size_t eol_length = match_eol_at(parser, start + ident_length);
if (eol_length) {
parser->current.end = start + ident_length + eol_length;
pm_newline_list_append(&parser->newline_list, parser->current.end - 1);
} else if (parser->end == (start + ident_length)) {
parser->current.end = start + ident_length;
if (newline == NULL) {
terminator_end = parser->end;
ident_end = parser->end;
at_end = true;
} else {
matched = false;
terminator_end++;
if (newline[-1] == '\r') {
ident_end--; // Remove \r
}
}

const uint8_t *terminator_start = ident_end - ident_length;
const uint8_t *cursor = start;

if (
lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_DASH ||
lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_TILDE
) {
while (cursor < terminator_start && pm_char_is_inline_whitespace(*cursor)) {
cursor++;
}
}

if (matched) {
if (
(cursor == terminator_start) &&
(memcmp(terminator_start, ident_start, ident_length) == 0)
) {
if (newline != NULL) {
pm_newline_list_append(&parser->newline_list, newline);
}
parser->current.end = terminator_end;
if (*lex_mode->as.heredoc.next_start == '\\') {
parser->next_start = NULL;
} else {
Expand All @@ -9794,7 +9813,7 @@ parser_lex(pm_parser_t *parser) {
LEX(PM_TOKEN_HEREDOC_END);
}
}

size_t whitespace = pm_heredoc_strspn_inline_whitespace(parser, &start, lex_mode->as.heredoc.indent);
if (
lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_TILDE &&
(lex_mode->as.heredoc.common_whitespace > whitespace) &&
Expand Down Expand Up @@ -9838,30 +9857,50 @@ parser_lex(pm_parser_t *parser) {
// If we have a - or ~ heredoc, then we can match after
// some leading whitespace.
const uint8_t *start = breakpoint + 1;
size_t whitespace = pm_heredoc_strspn_inline_whitespace(parser, &start, lex_mode->as.heredoc.indent);

// If we have hit a newline that is followed by a valid
// terminator, then we need to return the content of the
// heredoc here as string content. Then, the next time a
// token is lexed, it will match again and return the
// end of the heredoc.
if (
!was_escaped_newline &&
(start + ident_length <= parser->end) &&
(memcmp(start, ident_start, ident_length) == 0)
) {
// Heredoc terminators must be followed by a
// newline, CRLF, or EOF to be valid.
if (!was_escaped_newline && (start + ident_length <= parser->end)) {
// We want to match the terminator starting from the end of the line in case
// there is whitespace in the ident such as <<-' DOC' or <<~' DOC'.
const uint8_t *newline = next_newline(start, parser->end - start);

if (newline == NULL) {
newline = parser->end;
} else if (newline[-1] == '\r') {
newline--; // Remove \r
}

// Start of a possible terminator.
const uint8_t *terminator_start = newline - ident_length;

// Cursor to check for the leading whitespace. We skip the
// leading whitespace if we have a - or ~ heredoc.
const uint8_t *cursor = start;

if (lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_DASH ||
lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_TILDE) {
while (cursor < terminator_start && pm_char_is_inline_whitespace(*cursor)) {
cursor++;
}
}

if (
start + ident_length == parser->end ||
match_eol_at(parser, start + ident_length)
cursor == terminator_start &&
(memcmp(terminator_start, ident_start, ident_length) == 0)
) {
parser->current.end = breakpoint + 1;
pm_token_buffer_flush(parser, &token_buffer);
LEX(PM_TOKEN_STRING_CONTENT);
}
}

size_t whitespace = pm_heredoc_strspn_inline_whitespace(parser, &start, lex_mode->as.heredoc.indent);

// If we have hit a newline that is followed by a valid
// terminator, then we need to return the content of the
// heredoc here as string content. Then, the next time a
// token is lexed, it will match again and return the
// end of the heredoc.

if (lex_mode->as.heredoc.indent == PM_HEREDOC_INDENT_TILDE) {
if ((lex_mode->as.heredoc.common_whitespace > whitespace) && peek_at(parser, start) != '\n') {
lex_mode->as.heredoc.common_whitespace = whitespace;
Expand Down
19 changes: 19 additions & 0 deletions test/prism/fixtures/heredocs_leading_whitespace.txt
View file
@@ -0,0 +1,19 @@
<<-' FOO'
a
b
FOO

<<-' FOO'
a
b
FOO

<<~' FOO'
a
b
FOO

<<~' FOO'
a
b
FOO
7 changes: 7 additions & 0 deletions test/prism/locals_test.rb
View file
Expand Up @@ -68,6 +68,13 @@ class LocalsTest < TestCase
# HERE
todos << "seattlerb/heredoc_nested.txt"

# Ruby < 3.3.0 fails to parse:
#
# <<-' HERE'
# foo
# HERE
invalid << "heredocs_leading_whitespace.txt" if RUBY_VERSION < "3.3.0"

base = File.join(__dir__, "fixtures")
skips = invalid | todos

Expand Down
5 changes: 5 additions & 0 deletions test/prism/parse_test.rb
View file
Expand Up @@ -111,6 +111,11 @@ def test_parse_lex_file
# Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false.
ripper_should_parse = false if relative == "spanning_heredoc.txt"

# Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace charactes in the heredoc start.
# Example: <<~' EOF' or <<-' EOF'
# https://bugs.ruby-lang.org/issues/19539
ripper_should_parse = false if relative == "heredocs_leading_whitespace.txt" && RUBY_VERSION < "3.3.0"

define_method "test_filepath_#{relative}" do
# First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows,
# and explicitly set the external encoding to UTF-8 to override the binmode default.
Expand Down
49 changes: 49 additions & 0 deletions test/prism/snapshots/heredocs_leading_whitespace.txt
View file
@@ -0,0 +1,49 @@
@ ProgramNode (location: (1,0)-(16,10))
├── locals: []
└── statements:
@ StatementsNode (location: (1,0)-(16,10))
└── body: (length: 4)
├── @ StringNode (location: (1,0)-(1,10))
│ ├── flags: ∅
│ ├── opening_loc: (1,0)-(1,10) = "<<-' FOO'"
│ ├── content_loc: (2,0)-(4,0) = "a\nb\n"
│ ├── closing_loc: (4,0)-(5,0) = " FOO\n"
│ └── unescaped: "a\nb\n"
├── @ StringNode (location: (6,0)-(6,10))
│ ├── flags: ∅
│ ├── opening_loc: (6,0)-(6,10) = "<<-' FOO'"
│ ├── content_loc: (7,0)-(9,0) = "a\nb\n"
│ ├── closing_loc: (9,0)-(10,0) = " FOO\n"
│ └── unescaped: "a\nb\n"
├── @ InterpolatedStringNode (location: (11,0)-(11,10))
│ ├── opening_loc: (11,0)-(11,10) = "<<~' FOO'"
│ ├── parts: (length: 2)
│ │ ├── @ StringNode (location: (12,0)-(13,0))
│ │ │ ├── flags: ∅
│ │ │ ├── opening_loc: ∅
│ │ │ ├── content_loc: (12,0)-(13,0) = "a\n"
│ │ │ ├── closing_loc: ∅
│ │ │ └── unescaped: "a\n"
│ │ └── @ StringNode (location: (13,0)-(14,0))
│ │ ├── flags: ∅
│ │ ├── opening_loc: ∅
│ │ ├── content_loc: (13,0)-(14,0) = "b\n"
│ │ ├── closing_loc: ∅
│ │ └── unescaped: "b\n"
│ └── closing_loc: (14,0)-(15,0) = " FOO\n"
└── @ InterpolatedStringNode (location: (16,0)-(16,10))
├── opening_loc: (16,0)-(16,10) = "<<~' FOO'"
├── parts: (length: 2)
│ ├── @ StringNode (location: (17,0)-(18,0))
│ │ ├── flags: ∅
│ │ ├── opening_loc: ∅
│ │ ├── content_loc: (17,0)-(18,0) = "a\n"
│ │ ├── closing_loc: ∅
│ │ └── unescaped: "a\n"
│ └── @ StringNode (location: (18,0)-(19,0))
│ ├── flags: ∅
│ ├── opening_loc: ∅
│ ├── content_loc: (18,0)-(19,0) = "b\n"
│ ├── closing_loc: ∅
│ └── unescaped: "b\n"
└── closing_loc: (19,0)-(20,0) = " FOO\n"

0 comments on commit 562d949

Please sign in to comment.