Skip to content

Commit d14e56a

Browse files
committed
Scan every single characters in IRB::Color.scan
1 parent 8a074a6 commit d14e56a

File tree

2 files changed

+30
-27
lines changed

2 files changed

+30
-27
lines changed

lib/irb/color.rb

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,14 @@ def colorize_code(code, complete: true, ignore_error: false, colorable: colorabl
128128

129129
symbol_state = SymbolState.new
130130
colored = +''
131-
length = 0
132-
end_seen = false
133131

134132
scan(code, allow_last_error: !complete) do |token, str, expr|
133+
# handle uncolorable code
134+
if token.nil?
135+
colored << Reline::Unicode.escape_for_print(str)
136+
next
137+
end
138+
135139
# IRB::ColorPrinter skips colorizing fragments with any invalid token
136140
if ignore_error && ERROR_TOKENS.include?(token)
137141
return Reline::Unicode.escape_for_print(code)
@@ -147,15 +151,7 @@ def colorize_code(code, complete: true, ignore_error: false, colorable: colorabl
147151
colored << line
148152
end
149153
end
150-
length += str.bytesize
151-
end_seen = true if token == :on___end__
152-
end
153-
154-
# give up colorizing incomplete Ripper tokens
155-
unless end_seen or length == code.bytesize
156-
return Reline::Unicode.escape_for_print(code)
157154
end
158-
159155
colored
160156
end
161157

@@ -170,33 +166,38 @@ def without_circular_ref(obj, seen:, &block)
170166
end
171167

172168
def scan(code, allow_last_error:)
173-
pos = [1, 0]
174-
175169
verbose, $VERBOSE = $VERBOSE, nil
176170
RubyLex.compile_with_errors_suppressed(code) do |inner_code, line_no|
177171
lexer = Ripper::Lexer.new(inner_code, '(ripper)', line_no)
172+
byte_pos = 0
173+
line_positions = [0]
174+
inner_code.lines.each do |line|
175+
line_positions << line_positions.last + line.bytesize
176+
end
177+
178+
on_scan = proc do |elem|
179+
str = elem.tok
180+
start_pos = line_positions[elem.pos[0] - 1] + elem.pos[1]
181+
end_pos = start_pos + str.bytesize
182+
next if start_pos < byte_pos
183+
184+
yield(nil, inner_code.byteslice(byte_pos...start_pos), nil) if byte_pos < start_pos
185+
yield(elem.event, str, elem.state)
186+
byte_pos = end_pos
187+
end
188+
178189
if lexer.respond_to?(:scan) # Ruby 2.7+
179190
lexer.scan.each do |elem|
180-
str = elem.tok
181191
next if allow_last_error and /meets end of file|unexpected end-of-input/ =~ elem.message
182-
next if ([elem.pos[0], elem.pos[1] + str.bytesize] <=> pos) <= 0
183-
184-
str.each_line do |line|
185-
if line.end_with?("\n")
186-
pos[0] += 1
187-
pos[1] = 0
188-
else
189-
pos[1] += line.bytesize
190-
end
191-
end
192-
193-
yield(elem.event, str, elem.state)
192+
on_scan.call(elem)
194193
end
195194
else
196-
lexer.parse.each do |elem|
197-
yield(elem.event, elem.tok, elem.state)
195+
lexer.parse.sort_by(&:pos).each do |elem|
196+
on_scan.call(elem)
198197
end
199198
end
199+
# yield uncolorable DATA section
200+
yield(nil, inner_code.byteslice(byte_pos...inner_code.bytesize), nil) if byte_pos < inner_code.bytesize
200201
end
201202
ensure
202203
$VERBOSE = verbose

test/irb/test_color.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ def test_colorize_code
8888
"foo(*%W(bar))" => "foo(*#{RED}#{BOLD}%W(#{CLEAR}#{RED}bar#{CLEAR}#{RED}#{BOLD})#{CLEAR})",
8989
"$stdout" => "#{GREEN}#{BOLD}$stdout#{CLEAR}",
9090
"__END__" => "#{GREEN}__END__#{CLEAR}",
91+
"foo\n__END__\nbar" => "foo\n#{GREEN}__END__#{CLEAR}\nbar",
92+
"foo\n<<A\0\0bar\nA\nbaz" => "foo\n#{RED}<<A#{CLEAR}^@^@bar\n#{RED}A#{CLEAR}\nbaz",
9193
}
9294

9395
# specific to Ruby 2.7+

0 commit comments

Comments
 (0)