Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Deprecate RubyLex and warn about referencing to it #692

Merged
merged 1 commit into from
Aug 24, 2023
Merged

Deprecate RubyLex and warn about referencing to it #692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 8 additions & 3 deletions lib/irb/ruby-lex.rb
View file
Open in desktop
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
require "jruby" if RUBY_ENGINE == "jruby" require "jruby" if RUBY_ENGINE == "jruby"
require_relative "nesting_parser" require_relative "nesting_parser"


module IRB
# :stopdoc: # :stopdoc:
class RubyLex class RubyLex
ASSIGNMENT_NODE_TYPES = [ ASSIGNMENT_NODE_TYPES = [
Expand Down Expand Up @@ -153,7 +154,7 @@ def prompt(opens, continue, line_num_offset)


def check_code_state(code) def check_code_state(code)
tokens = self.class.ripper_lex_without_warning(code, context: @context) tokens = self.class.ripper_lex_without_warning(code, context: @context)
opens = IRB::NestingParser.open_tokens(tokens) opens = NestingParser.open_tokens(tokens)
[tokens, opens, code_terminated?(code, tokens, opens)] [tokens, opens, code_terminated?(code, tokens, opens)]
end end


Expand Down Expand Up @@ -349,7 +350,7 @@ def indent_difference(lines, line_results, line_index)
end end


def process_indent_level(tokens, lines, line_index, is_newline) def process_indent_level(tokens, lines, line_index, is_newline)
line_results = IRB::NestingParser.parse_by_line(tokens) line_results = NestingParser.parse_by_line(tokens)
result = line_results[line_index] result = line_results[line_index]
if result if result
_tokens, prev_opens, next_opens, min_depth = result _tokens, prev_opens, next_opens, min_depth = result
Expand Down Expand Up @@ -484,7 +485,7 @@ def check_termination_in_prev_line(code)
if first_token && first_token.state != Ripper::EXPR_DOT if first_token && first_token.state != Ripper::EXPR_DOT
tokens_without_last_line = tokens[0..index] tokens_without_last_line = tokens[0..index]
code_without_last_line = tokens_without_last_line.map(&:tok).join code_without_last_line = tokens_without_last_line.map(&:tok).join
opens_without_last_line = IRB::NestingParser.open_tokens(tokens_without_last_line) opens_without_last_line = NestingParser.open_tokens(tokens_without_last_line)
if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line) if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line)
return last_line_tokens.map(&:tok).join return last_line_tokens.map(&:tok).join
end end
Expand All @@ -494,3 +495,7 @@ def check_termination_in_prev_line(code)
end end
end end
# :startdoc: # :startdoc:
end

RubyLex = IRB::RubyLex
Object.deprecate_constant(:RubyLex)
4 changes: 2 additions & 2 deletions test/irb/test_nesting_parser.rb
View file
Open in desktop
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def teardown
end end


def parse_by_line(code) def parse_by_line(code)
IRB::NestingParser.parse_by_line(RubyLex.ripper_lex_without_warning(code)) IRB::NestingParser.parse_by_line(IRB::RubyLex.ripper_lex_without_warning(code))
end end


def test_open_tokens def test_open_tokens
Expand All @@ -27,7 +27,7 @@ def f
x: " x: "
#{p(1, 2, 3 #{p(1, 2, 3
EOS EOS
opens = IRB::NestingParser.open_tokens(RubyLex.ripper_lex_without_warning(code)) opens = IRB::NestingParser.open_tokens(IRB::RubyLex.ripper_lex_without_warning(code))
assert_equal(%w[class def if do { " #{ (], opens.map(&:tok)) assert_equal(%w[class def if do { " #{ (], opens.map(&:tok))
end end


Expand Down
24 changes: 16 additions & 8 deletions test/irb/test_ruby_lex.rb
View file
Open in desktop
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def test_interpolate_token_with_heredoc_and_unclosed_embexpr
#{⑤&<<C|⑥ #{⑤&<<C|⑥
EOC EOC
ripper_tokens = Ripper.tokenize(code) ripper_tokens = Ripper.tokenize(code)
rubylex_tokens = RubyLex.ripper_lex_without_warning(code) rubylex_tokens = IRB::RubyLex.ripper_lex_without_warning(code)
# Assert no missing part # Assert no missing part
assert_equal(code, rubylex_tokens.map(&:tok).join) assert_equal(code, rubylex_tokens.map(&:tok).join)
# Assert ripper tokens are not removed # Assert ripper tokens are not removed
Expand Down Expand Up @@ -83,7 +83,7 @@ def test_code_block_open_with_should_continue
end end


def test_broken_percent_literal def test_broken_percent_literal
tokens = RubyLex.ripper_lex_without_warning('%wwww') tokens = IRB::RubyLex.ripper_lex_without_warning('%wwww')
pos_to_index = {} pos_to_index = {}
tokens.each_with_index { |t, i| tokens.each_with_index { |t, i|
assert_nil(pos_to_index[t.pos], "There is already another token in the position of #{t.inspect}.") assert_nil(pos_to_index[t.pos], "There is already another token in the position of #{t.inspect}.")
Expand All @@ -92,7 +92,7 @@ def test_broken_percent_literal
end end


def test_broken_percent_literal_in_method def test_broken_percent_literal_in_method
tokens = RubyLex.ripper_lex_without_warning(<<~EOC.chomp) tokens = IRB::RubyLex.ripper_lex_without_warning(<<~EOC.chomp)
def foo def foo
%wwww %wwww
end end
Expand All @@ -106,7 +106,7 @@ def foo


def test_unterminated_code def test_unterminated_code
['do', '<<A'].each do |code| ['do', '<<A'].each do |code|
tokens = RubyLex.ripper_lex_without_warning(code) tokens = IRB::RubyLex.ripper_lex_without_warning(code)
assert_equal(code, tokens.map(&:tok).join, "Cannot reconstruct code from tokens") assert_equal(code, tokens.map(&:tok).join, "Cannot reconstruct code from tokens")
error_tokens = tokens.map(&:event).grep(/error/) error_tokens = tokens.map(&:event).grep(/error/)
assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token') assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token')
Expand All @@ -115,7 +115,7 @@ def test_unterminated_code


def test_unterminated_heredoc_string_literal def test_unterminated_heredoc_string_literal
['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code| ['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
tokens = RubyLex.ripper_lex_without_warning(code) tokens = IRB::RubyLex.ripper_lex_without_warning(code)
string_literal = IRB::NestingParser.open_tokens(tokens).last string_literal = IRB::NestingParser.open_tokens(tokens).last
assert_equal('<<A', string_literal&.tok) assert_equal('<<A', string_literal&.tok)
end end
Expand Down Expand Up @@ -150,7 +150,7 @@ def test_indent_level_with_heredoc_and_embdoc


def test_assignment_expression def test_assignment_expression
context = build_context context = build_context
ruby_lex = RubyLex.new(context) ruby_lex = IRB::RubyLex.new(context)


[ [
"foo = bar", "foo = bar",
Expand Down Expand Up @@ -194,14 +194,22 @@ def test_assignment_expression


def test_assignment_expression_with_local_variable def test_assignment_expression_with_local_variable
context = build_context context = build_context
ruby_lex = RubyLex.new(context) ruby_lex = IRB::RubyLex.new(context)
code = "a /1;x=1#/" code = "a /1;x=1#/"
refute(ruby_lex.assignment_expression?(code), "#{code}: should not be an assignment expression") refute(ruby_lex.assignment_expression?(code), "#{code}: should not be an assignment expression")
context.workspace.binding.eval('a = 1') context.workspace.binding.eval('a = 1')
assert(ruby_lex.assignment_expression?(code), "#{code}: should be an assignment expression") assert(ruby_lex.assignment_expression?(code), "#{code}: should be an assignment expression")
refute(ruby_lex.assignment_expression?(""), "empty code should not be an assignment expression") refute(ruby_lex.assignment_expression?(""), "empty code should not be an assignment expression")
end end


def test_initialising_the_old_top_level_ruby_lex
_, err = capture_output do
::RubyLex.new(nil)
end

assert_match(/warning: constant ::RubyLex is deprecated/, err)
end

private private


def build_context(local_variables = nil) def build_context(local_variables = nil)
Expand Down Expand Up @@ -239,7 +247,7 @@ def assert_code_block_open(lines, expected, local_variables: [])
def check_state(lines, local_variables: []) def check_state(lines, local_variables: [])
context = build_context(local_variables) context = build_context(local_variables)
code = lines.map { |l| "#{l}\n" }.join # code should end with "\n" code = lines.map { |l| "#{l}\n" }.join # code should end with "\n"
ruby_lex = RubyLex.new(context) ruby_lex = IRB::RubyLex.new(context)
tokens, opens, terminated = ruby_lex.check_code_state(code) tokens, opens, terminated = ruby_lex.check_code_state(code)
indent_level = ruby_lex.calc_indent_level(opens) indent_level = ruby_lex.calc_indent_level(opens)
continue = ruby_lex.should_continue?(tokens) continue = ruby_lex.should_continue?(tokens)
Expand Down