Merge pull request #163 from aycabta/use-ripper-lexer-scan-to-take-br…

…oken-tokens

Use Ripper::Lexer#scan to take broken tokens

lib/irb/ruby-lex.rb

@@ -106,11 +106,38 @@ def set_prompt(p = nil, &block)
     end
   end
 
+  ERROR_TOKENS = [
+    :on_parse_error,
+    :compile_error,
+    :on_assign_error,
+    :on_alias_error,
+    :on_class_name_error,
+    :on_param_error
+  ]
+
   def ripper_lex_without_warning(code)
     verbose, $VERBOSE = $VERBOSE, nil
     tokens = nil
     self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
-      tokens = Ripper.lex(inner_code, '-', line_no)
+      lexer = Ripper::Lexer.new(inner_code, '-', line_no)
+      if lexer.respond_to?(:scan) # Ruby 2.7+
+        tokens = []
+        pos_to_index = {}
+        lexer.scan.each do |t|
+          if pos_to_index.has_key?(t[0])
+            index = pos_to_index[t[0]]
+            found_tk = tokens[index]
+            if ERROR_TOKENS.include?(found_tk[1]) && !ERROR_TOKENS.include?(t[1])
+              tokens[index] = t
+            end
+          else
+            pos_to_index[t[0]] = tokens.size
+            tokens << t
+          end
+        end
+      else
+        tokens = lexer.parse
+      end
     end
     $VERBOSE = verbose
     tokens

test/irb/test_ruby_lex.rb

@@ -367,6 +367,25 @@ def test_oneliner_def_in_multiple_lines
       end
     end
 
+    def test_broken_heredoc
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        skip 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
+      input_with_correct_indents = [
+        Row.new(%q(def foo), nil, 2, 1),
+        Row.new(%q(  <<~Q), nil, 2, 1),
+        Row.new(%q(  Qend), nil, 2, 1),
+      ]
+
+      lines = []
+      input_with_correct_indents.each do |row|
+        lines << row.content
+        assert_indenting(lines, row.current_line_spaces, false)
+        assert_indenting(lines, row.new_line_spaces, true)
+        assert_nesting_level(lines, row.nesting_level)
+      end
+    end
+
     PromptRow = Struct.new(:prompt, :content)
 
     class MockIO_DynamicPrompt
@@ -423,5 +442,37 @@ def test_dyanmic_prompt_with_blank_line
       expected_prompt_list = input_with_prompt.map(&:prompt)
       assert_dynamic_prompt(lines, expected_prompt_list)
     end
+
+    def test_broken_percent_literal
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        skip 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
+
+      ruby_lex = RubyLex.new
+      tokens = ruby_lex.ripper_lex_without_warning('%wwww')
+      pos_to_index = {}
+      tokens.each_with_index { |t, i|
+        assert_nil(pos_to_index[t[0]], "There is already another token in the position of #{t.inspect}.")
+        pos_to_index[t[0]] = i
+      }
+    end
+
+    def test_broken_percent_literal_in_method
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        skip 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
+
+      ruby_lex = RubyLex.new
+      tokens = ruby_lex.ripper_lex_without_warning(<<~EOC.chomp)
+        def foo
+          %wwww
+        end
+      EOC
+      pos_to_index = {}
+      tokens.each_with_index { |t, i|
+        assert_nil(pos_to_index[t[0]], "There is already another token in the position of #{t.inspect}.")
+        pos_to_index[t[0]] = i
+      }
+    end
   end
 end