Fix up test suite

Author: kddnewton

rakelib/test.rake

@@ -5,7 +5,7 @@ require "rake/testtask"
 config = lambda do |t|
   t.libs << "test"
   t.libs << "lib"
-  t.test_files = FileList["test/prism/parse_stream_test.rb"]
+  t.test_files = FileList["test/**/*_test.rb"]
 end
 
 Rake::TestTask.new(:test, &config)

src/prism.c

@@ -8831,6 +8831,8 @@ pm_token_buffer_escape(pm_parser_t *parser, pm_token_buffer_t *token_buffer) {
 
     const uint8_t *end = parser->current.end - 1;
     pm_buffer_append_bytes(&token_buffer->buffer, start, (size_t) (end - start));
+
+    token_buffer->cursor = end;
 }
 
 /**

test/prism/parser_test.rb

@@ -45,22 +45,22 @@ class ParserTest < TestCase
     base = File.join(__dir__, "fixtures")
 
     # These files are erroring because of the parser gem being wrong.
-    skip_incorrect = %w[
-      embdoc_no_newline_at_end.txt
+    skip_incorrect = [
+      "embdoc_no_newline_at_end.txt"
     ]
 
     # These files are either failing to parse or failing to translate, so we'll
     # skip them for now.
-    skip_all = skip_incorrect | %w[
-      dash_heredocs.txt
-      dos_endings.txt
-      heredocs_with_ignored_newlines.txt
-      regex.txt
-      regex_char_width.txt
-      spanning_heredoc.txt
-      spanning_heredoc_newlines.txt
-      tilde_heredocs.txt
-      unescaping.txt
+    skip_all = skip_incorrect | [
+      "dash_heredocs.txt",
+      "dos_endings.txt",
+      "heredocs_with_ignored_newlines.txt",
+      "regex.txt",
+      "regex_char_width.txt",
+      "spanning_heredoc.txt",
+      "spanning_heredoc_newlines.txt",
+      "tilde_heredocs.txt",
+      "unescaping.txt"
     ]
 
     # Not sure why these files are failing on JRuby, but skipping them for now.
@@ -70,21 +70,21 @@ class ParserTest < TestCase
 
     # These files are failing to translate their lexer output into the lexer
     # output expected by the parser gem, so we'll skip them for now.
-    skip_tokens = %w[
-      comments.txt
-      constants.txt
-      endless_range_in_conditional.txt
-      heredoc_with_comment.txt
-      heredoc_with_escaped_newline_at_start.txt
-      heredocs_leading_whitespace.txt
-      heredocs_nested.txt
-      heredocs_with_ignored_newlines_and_non_empty.txt
-      indented_file_end.txt
-      non_alphanumeric_methods.txt
-      range_begin_open_inclusive.txt
-      single_quote_heredocs.txt
-      strings.txt
-      xstring.txt
+    skip_tokens = [
+      "comments.txt",
+      "constants.txt",
+      "endless_range_in_conditional.txt",
+      "heredoc_with_comment.txt",
+      "heredoc_with_escaped_newline_at_start.txt",
+      "heredocs_leading_whitespace.txt",
+      "heredocs_nested.txt",
+      "heredocs_with_ignored_newlines_and_non_empty.txt",
+      "indented_file_end.txt",
+      "non_alphanumeric_methods.txt",
+      "range_begin_open_inclusive.txt",
+      "single_quote_heredocs.txt",
+      "strings.txt",
+      "xstring.txt"
     ]
 
     Dir["*.txt", base: base].each do |name|

test/prism/ripper_test.rb

@@ -25,30 +25,29 @@ class RipperTest < TestCase
       # Ripper cannot handle named capture groups in regular expressions.
       "regex.txt",
       "regex_char_width.txt",
-      "whitequark/lvar_injecting_match.txt"
+      "whitequark/lvar_injecting_match.txt",
+
+      # Ripper fails to understand some structures that span across heredocs.
+      "spanning_heredoc.txt"
     ]
 
     omitted = [
       "dos_endings.txt",
       "heredocs_with_ignored_newlines.txt",
+      "seattlerb/block_call_dot_op2_brace_block.txt",
+      "seattlerb/block_command_operation_colon.txt",
+      "seattlerb/block_command_operation_dot.txt",
       "seattlerb/heredoc__backslash_dos_format.txt",
       "seattlerb/heredoc_backslash_nl.txt",
       "seattlerb/heredoc_nested.txt",
       "seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt",
-      "seattlerb/heredoc_squiggly_no_indent.txt",
-      "spanning_heredoc.txt",
       "tilde_heredocs.txt",
       "unparser/corpus/semantic/dstr.txt",
       "whitequark/dedenting_heredoc.txt",
-      "whitequark/parser_bug_640.txt",
       "whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt",
       "whitequark/parser_slash_slash_n_escaping_in_literals.txt",
-      "whitequark/slash_newline_in_heredocs.txt",
-
-      "seattlerb/block_call_dot_op2_brace_block.txt",
-      "seattlerb/block_command_operation_colon.txt",
-      "seattlerb/block_command_operation_dot.txt",
-      "whitequark/send_block_chain_cmd.txt"
+      "whitequark/send_block_chain_cmd.txt",
+      "whitequark/slash_newline_in_heredocs.txt"
     ]
 
     relatives.each do |relative|