Handle CLRF inside heredoc contents

Author: kddnewton

src/prism.c

@@ -11267,11 +11267,11 @@ parser_lex(pm_parser_t *parser) {
             // Otherwise we'll be parsing string content. These are the places
             // where we need to split up the content of the heredoc. We'll use
             // strpbrk to find the first of these characters.
-            uint8_t breakpoints[] = "\n\\#";
+            uint8_t breakpoints[] = "\r\n\\#";
 
             pm_heredoc_quote_t quote = lex_mode->as.heredoc.quote;
             if (quote == PM_HEREDOC_QUOTE_SINGLE) {
-                breakpoints[2] = '\0';
+                breakpoints[3] = '\0';
             }
 
             const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
@@ -11285,6 +11285,21 @@ parser_lex(pm_parser_t *parser) {
                         parser->current.end = breakpoint + 1;
                         breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
                         break;
+                    case '\r':
+                        parser->current.end = breakpoint + 1;
+
+                        if (peek_at(parser, breakpoint + 1) != '\n') {
+                            breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
+                            break;
+                        }
+
+                        // If we hit a \r\n sequence, then we want to replace it
+                        // with a single \n character in the final string.
+                        pm_token_buffer_escape(parser, &token_buffer);
+                        breakpoint++;
+                        token_buffer.cursor = breakpoint;
+
+                        /* fallthrough */
                     case '\n': {
                         if (parser->heredoc_end != NULL && (parser->heredoc_end > breakpoint)) {
                             parser_flush_heredoc_end(parser);

test/prism/ruby_parser_test.rb

@@ -52,25 +52,10 @@ class RubyParserTest < TestCase
       whitequark/string_concat.txt
     ]
 
-    # These files contain CRLF line endings, which ruby_parser translates into
-    # LF before it gets back to the node. This means the node actually has the
-    # wrong contents.
-    crlf = %w[
-      dos_endings.txt
-      heredoc_with_comment.txt
-      seattlerb/heredoc__backslash_dos_format.txt
-      seattlerb/heredoc_with_carriage_return_escapes_windows.txt
-      seattlerb/heredoc_with_extra_carriage_horrible_mix.txt
-      seattlerb/heredoc_with_extra_carriage_returns_windows.txt
-      seattlerb/heredoc_with_extra_carriage_returns.txt
-      seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt
-      seattlerb/heredoc_with_only_carriage_returns_windows.txt
-      seattlerb/heredoc_with_only_carriage_returns.txt
-    ]
-
     # https://github.com/seattlerb/ruby_parser/issues/344
-    failures = crlf | %w[
+    failures = %w[
       alias.txt
+      dos_endings.txt
       heredocs_with_ignored_newlines.txt
       method_calls.txt
       methods.txt
@@ -79,8 +64,10 @@ class RubyParserTest < TestCase
       patterns.txt
       regex.txt
       seattlerb/and_multi.txt
+      seattlerb/heredoc__backslash_dos_format.txt
       seattlerb/heredoc_bad_hex_escape.txt
       seattlerb/heredoc_bad_oct_escape.txt
+      seattlerb/heredoc_with_extra_carriage_horrible_mix.txt
       spanning_heredoc_newlines.txt
       spanning_heredoc.txt
       tilde_heredocs.txt