Skip to content

Commit 44bc712

Browse files
tompngst0012
andauthored
Fix code terminated check with heredoc and backtick (#390)
* Fix backtick method def method call handled as backtick open * Fix handling heredoc in check_string_literal * Sort result of lexer.parse by pos in ruby<2.7. It's not sorted when the given code includes heredoc. * Update lib/irb/ruby-lex.rb Co-authored-by: Stan Lo <stan001212@gmail.com> * Update lib/irb/ruby-lex.rb Co-authored-by: Stan Lo <stan001212@gmail.com> * Add check_string_literal test for heredoc code that does not end with newline Co-authored-by: Stan Lo <stan001212@gmail.com>
1 parent f6f2b80 commit 44bc712

File tree

2 files changed

+58
-6
lines changed

2 files changed

+58
-6
lines changed

lib/irb/ruby-lex.rb

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ def self.ripper_lex_without_warning(code, context: nil)
162162
end
163163
end
164164
else
165-
lexer.parse.reject { |it| it.pos.first == 0 }
165+
lexer.parse.reject { |it| it.pos.first == 0 }.sort_by(&:pos)
166166
end
167167
end
168168
ensure
@@ -706,6 +706,7 @@ def check_string_literal(tokens)
706706
i = 0
707707
start_token = []
708708
end_type = []
709+
pending_heredocs = []
709710
while i < tokens.size
710711
t = tokens[i]
711712
case t.event
@@ -729,18 +730,27 @@ def check_string_literal(tokens)
729730
end
730731
end
731732
when :on_backtick
732-
start_token << t
733-
end_type << :on_tstring_end
733+
if t.state.allbits?(Ripper::EXPR_BEG)
734+
start_token << t
735+
end_type << :on_tstring_end
736+
end
734737
when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
735738
start_token << t
736739
end_type << :on_tstring_end
737740
when :on_heredoc_beg
738-
start_token << t
739-
end_type << :on_heredoc_end
741+
pending_heredocs << t
742+
end
743+
744+
if pending_heredocs.any? && t.tok.include?("\n")
745+
pending_heredocs.reverse_each do |t|
746+
start_token << t
747+
end_type << :on_heredoc_end
748+
end
749+
pending_heredocs = []
740750
end
741751
i += 1
742752
end
743-
start_token.last.nil? ? nil : start_token.last
753+
pending_heredocs.first || start_token.last
744754
end
745755

746756
def process_literal_type(tokens = @tokens)

test/irb/test_ruby_lex.rb

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,40 @@ def test_endless_range_at_end_of_line
170170
assert_dynamic_prompt(lines, expected_prompt_list)
171171
end
172172

173+
def test_heredoc_with_embexpr
174+
input_with_prompt = [
175+
PromptRow.new('001:0:":* ', %q(<<A+%W[#{<<B)),
176+
PromptRow.new('002:0:":* ', %q(#{<<C+%W[)),
177+
PromptRow.new('003:0:":* ', %q()),
178+
PromptRow.new('004:0:":* ', %q(C)),
179+
PromptRow.new('005:0:]:* ', %q()),
180+
PromptRow.new('006:0:":* ', %q(]})),
181+
PromptRow.new('007:0:":* ', %q(})),
182+
PromptRow.new('008:0:":* ', %q(A)),
183+
PromptRow.new('009:0:]:* ', %q(B)),
184+
PromptRow.new('010:0:]:* ', %q(})),
185+
PromptRow.new('011:0: :> ', %q(])),
186+
PromptRow.new('012:0: :* ', %q()),
187+
]
188+
189+
lines = input_with_prompt.map(&:content)
190+
expected_prompt_list = input_with_prompt.map(&:prompt)
191+
assert_dynamic_prompt(lines, expected_prompt_list)
192+
end
193+
194+
def test_backtick_method
195+
input_with_prompt = [
196+
PromptRow.new('001:0: :> ', %q(self.`(arg))),
197+
PromptRow.new('002:0: :* ', %q()),
198+
PromptRow.new('003:0: :> ', %q(def `(); end)),
199+
PromptRow.new('004:0: :* ', %q()),
200+
]
201+
202+
lines = input_with_prompt.map(&:content)
203+
expected_prompt_list = input_with_prompt.map(&:prompt)
204+
assert_dynamic_prompt(lines, expected_prompt_list)
205+
end
206+
173207
def test_incomplete_coding_magic_comment
174208
input_with_correct_indents = [
175209
Row.new(%q(#coding:u), nil, 0),
@@ -632,5 +666,13 @@ def test_unterminated_code
632666
assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token')
633667
end
634668
end
669+
670+
def test_unterminated_heredoc_string_literal
671+
['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
672+
tokens = RubyLex.ripper_lex_without_warning(code)
673+
string_literal = RubyLex.new.check_string_literal(tokens)
674+
assert_equal('<<A', string_literal&.tok)
675+
end
676+
end
635677
end
636678
end

0 commit comments

Comments
 (0)