Skip to content

Commit

Permalink
Handle chomped bytesize with lines without newlines
Browse files Browse the repository at this point in the history
  • Loading branch information
kddnewton committed Jun 7, 2024
1 parent 89fb586 commit d268507
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 3 deletions.
11 changes: 9 additions & 2 deletions lib/prism/translation/parser/compiler.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2040,6 +2040,13 @@ def visit_block(call, block)
end
end

# The parser gem automatically converts \r\n to \n, meaning our offsets
# need to be adjusted to always subtract 1 from the length.
def chomped_bytesize(line)
chomped = line.chomp
chomped.bytesize + (chomped == line ? 0 : 1)
end

# Visit a heredoc that can be either a string or an xstring.
def visit_heredoc(node)
children = Array.new
Expand All @@ -2066,14 +2073,14 @@ def visit_heredoc(node)
if node.opening.end_with?("'")
escaped.each do |line|
escaped_lengths << line.bytesize
normalized_lengths << (line.chomp.bytesize + 1)
normalized_lengths << chomped_bytesize(line)
end
else
escaped
.chunk_while { |before, after| before.match?(/(?<!\\)\\\r?\n$/) }
.each do |lines|
escaped_lengths << lines.sum(&:bytesize)
normalized_lengths << lines.sum { |line| line.chomp.bytesize + 1 }
normalized_lengths << lines.sum { |line| chomped_bytesize(line) }
end
end

Expand Down
2 changes: 1 addition & 1 deletion test/prism/ruby/parser_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ class ParserTest < TestCase
# These files are either failing to parse or failing to translate, so we'll
# skip them for now.
skip_all = skip_incorrect | [
"dash_heredocs.txt",
"regex.txt",
"regex_char_width.txt",
"unescaping.txt",
Expand All @@ -86,6 +85,7 @@ class ParserTest < TestCase
# output expected by the parser gem, so we'll skip them for now.
skip_tokens = [
"comments.txt",
"dash_heredocs.txt",
"dos_endings.txt",
"embdoc_no_newline_at_end.txt",
"heredoc_with_comment.txt",
Expand Down

0 comments on commit d268507

Please sign in to comment.