Use Ripper::Lexer#scan to take broken tokens

ref. ruby/reline#242

lib/irb/ruby-lex.rb

@@ -66,6 +66,7 @@ def set_input(io, p = nil, &block)
         unprocessed_tokens = []
         line_num_offset = 0
         tokens.each do |t|
+          next if t[1] == :on_parse_error || t[1] == :compile_error
           partial_tokens << t
           unprocessed_tokens << t
           if t[2].include?("\n")
@@ -110,7 +111,12 @@ def ripper_lex_without_warning(code)
     verbose, $VERBOSE = $VERBOSE, nil
     tokens = nil
     self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
-      tokens = Ripper.lex(inner_code, '-', line_no)
+      lexer = Ripper::Lexer.new(inner_code, '-', line_no)
+      if lexer.respond_to?(:scan) # Ruby 2.7+
+        tokens = lexer.scan
+      else
+        tokens = lexer.parse
+      end
     end
     $VERBOSE = verbose
     tokens
@@ -122,6 +128,7 @@ def find_prev_spaces(line_index)
     prev_spaces = md.nil? ? 0 : md[1].count(' ')
     line_count = 0
     @tokens.each_with_index do |t, i|
+      next if t[1] == :on_parse_error || t[1] == :compile_error
       if t[2].include?("\n")
         line_count += t[2].count("\n")
         if line_count >= line_index
@@ -350,6 +357,7 @@ def process_nesting_level(tokens = @tokens)
     indent = 0
     in_oneliner_def = nil
     tokens.each_with_index { |t, index|
+      next if t[1] == :on_parse_error || t[1] == :compile_error
       # detecting one-liner method definition
       if in_oneliner_def.nil?
         if t[3].allbits?(Ripper::EXPR_ENDFN)
@@ -435,6 +443,7 @@ def check_newline_depth_difference
     open_brace_on_line = 0
     in_oneliner_def = nil
     @tokens.each_with_index do |t, index|
+      next if t[1] == :on_parse_error || t[1] == :compile_error
       # detecting one-liner method definition
       if in_oneliner_def.nil?
         if t[3].allbits?(Ripper::EXPR_ENDFN)
@@ -504,6 +513,7 @@ def check_corresponding_token_depth
     open_brace_on_line = 0
     in_oneliner_def = nil
     @tokens.each_with_index do |t, index|
+      next if t[1] == :on_parse_error || t[1] == :compile_error
       # detecting one-liner method definition
       if in_oneliner_def.nil?
         if t[3].allbits?(Ripper::EXPR_ENDFN)

test/irb/test_ruby_lex.rb

@@ -367,6 +367,25 @@ def test_oneliner_def_in_multiple_lines
       end
     end
 
+    def test_broken_heredoc
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        skip 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
+      input_with_correct_indents = [
+        Row.new(%q(def foo), nil, 2, 1),
+        Row.new(%q(  <<~Q), nil, 2, 1),
+        Row.new(%q(  Qend), nil, 2, 1),
+      ]
+
+      lines = []
+      input_with_correct_indents.each do |row|
+        lines << row.content
+        assert_indenting(lines, row.current_line_spaces, false)
+        assert_indenting(lines, row.new_line_spaces, true)
+        assert_nesting_level(lines, row.nesting_level)
+      end
+    end
+
     PromptRow = Struct.new(:prompt, :content)
 
     class MockIO_DynamicPrompt