[Bug #19838] Flush delayed token nonconsecutive with the next token

ruby · Dec 1, 2023 · e5e1f98 · e5e1f98
1 parent add0ab0
commit e5e1f98
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 0 deletions.
diff --git a/parse.y b/parse.y
@@ -7631,6 +7631,14 @@ add_delayed_token(struct parser_params *p, const char *tok, const char *end, int
 #endif
 
     if (tok < end) {
+        if (has_delayed_token(p)) {
+            bool next_line = end_with_newline_p(p, p->delayed.token);
+            int end_line = (next_line ? 1 : 0) + p->delayed.end_line;
+            int end_col = (next_line ? 0 : p->delayed.end_col);
+            if (end_line != p->ruby_sourceline || end_col != tok - p->lex.pbeg) {
+                dispatch_delayed_token(p, tSTRING_CONTENT);
+            }
+        }
         if (!has_delayed_token(p)) {
             p->delayed.token = rb_str_buf_new(end - tok);
             rb_enc_associate(p->delayed.token, p->enc);

diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
@@ -273,4 +273,65 @@ def test_heredoc_unterminated_interpolation
 
     assert_include(Ripper.tokenize(code).join(""), "+1")
   end
+
+  def test_nested_heredoc
+    code = <<~'HEREDOC'
+    <<~H1
+      1
+      #{<<~H2}
+        2
+      H2
+      3
+    H1
+    HEREDOC
+
+    expected = [
+      [[1, 0], :on_heredoc_beg, "<<~H1", state(:EXPR_BEG)],
+      [[1, 5], :on_nl, "\n", state(:EXPR_BEG)],
+      [[2, 0], :on_ignored_sp, "  ", state(:EXPR_BEG)],
+      [[2, 2], :on_tstring_content, "1\n", state(:EXPR_BEG)],
+      [[3, 0], :on_ignored_sp, "  ", state(:EXPR_BEG)],
+      [[3, 2], :on_embexpr_beg, "\#{", state(:EXPR_BEG)],
+      [[3, 4], :on_heredoc_beg, "<<~H2", state(:EXPR_BEG)],
+      [[3, 9], :on_embexpr_end, "}", state(:EXPR_END)],
+      [[3, 10], :on_tstring_content, "\n", state(:EXPR_BEG)],
+      [[4, 0], :on_ignored_sp, "    ", state(:EXPR_BEG)],
+      [[4, 4], :on_tstring_content, "2\n", state(:EXPR_BEG)],
+      [[5, 0], :on_heredoc_end, "  H2\n", state(:EXPR_BEG)],
+      [[6, 0], :on_ignored_sp, "  ", state(:EXPR_BEG)],
+      [[6, 2], :on_tstring_content, "3\n", state(:EXPR_BEG)],
+      [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)],
+    ]
+    assert_equal(code, Ripper.tokenize(code).join(""))
+    assert_equal(expected, result = Ripper.lex(code),
+                 proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}})
+
+    code = <<~'HEREDOC'
+    <<-H1
+      1
+      #{<<~H2}
+        2
+      H2
+      3
+    H1
+    HEREDOC
+
+    expected = [
+      [[1, 0], :on_heredoc_beg, "<<-H1", state(:EXPR_BEG)],
+      [[1, 5], :on_nl, "\n", state(:EXPR_BEG)],
+      [[2, 0], :on_tstring_content, "  1\n  ", state(:EXPR_BEG)],
+      [[3, 2], :on_embexpr_beg, "\#{", state(:EXPR_BEG)],
+      [[3, 4], :on_heredoc_beg, "<<~H2", state(:EXPR_BEG)],
+      [[3, 9], :on_embexpr_end, "}", state(:EXPR_END)],
+      [[3, 10], :on_tstring_content, "\n", state(:EXPR_BEG)],
+      [[4, 0], :on_ignored_sp, "    ", state(:EXPR_BEG)],
+      [[4, 4], :on_tstring_content, "2\n", state(:EXPR_BEG)],
+      [[5, 0], :on_heredoc_end, "  H2\n", state(:EXPR_BEG)],
+      [[6, 0], :on_tstring_content, "  3\n", state(:EXPR_BEG)],
+      [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)],
+    ]
+    assert_equal(code, Ripper.tokenize(code).join(""))
+    assert_equal(expected, result = Ripper.lex(code),
+                 proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}})
+  end
 end