@@ -128,10 +128,14 @@ def colorize_code(code, complete: true, ignore_error: false, colorable: colorabl
128
128
129
129
symbol_state = SymbolState . new
130
130
colored = +''
131
- length = 0
132
- end_seen = false
133
131
134
132
scan ( code , allow_last_error : !complete ) do |token , str , expr |
133
+ # handle uncolorable code
134
+ if token . nil?
135
+ colored << Reline ::Unicode . escape_for_print ( str )
136
+ next
137
+ end
138
+
135
139
# IRB::ColorPrinter skips colorizing fragments with any invalid token
136
140
if ignore_error && ERROR_TOKENS . include? ( token )
137
141
return Reline ::Unicode . escape_for_print ( code )
@@ -147,15 +151,7 @@ def colorize_code(code, complete: true, ignore_error: false, colorable: colorabl
147
151
colored << line
148
152
end
149
153
end
150
- length += str . bytesize
151
- end_seen = true if token == :on___end__
152
- end
153
-
154
- # give up colorizing incomplete Ripper tokens
155
- unless end_seen or length == code . bytesize
156
- return Reline ::Unicode . escape_for_print ( code )
157
154
end
158
-
159
155
colored
160
156
end
161
157
@@ -170,33 +166,38 @@ def without_circular_ref(obj, seen:, &block)
170
166
end
171
167
172
168
def scan ( code , allow_last_error :)
173
- pos = [ 1 , 0 ]
174
-
175
169
verbose , $VERBOSE = $VERBOSE, nil
176
170
RubyLex . compile_with_errors_suppressed ( code ) do |inner_code , line_no |
177
171
lexer = Ripper ::Lexer . new ( inner_code , '(ripper)' , line_no )
172
+ byte_pos = 0
173
+ line_positions = [ 0 ]
174
+ inner_code . lines . each do |line |
175
+ line_positions << line_positions . last + line . bytesize
176
+ end
177
+
178
+ on_scan = proc do |elem |
179
+ str = elem . tok
180
+ start_pos = line_positions [ elem . pos [ 0 ] - 1 ] + elem . pos [ 1 ]
181
+ end_pos = start_pos + str . bytesize
182
+ next if start_pos < byte_pos
183
+
184
+ yield ( nil , inner_code . byteslice ( byte_pos ...start_pos ) , nil ) if byte_pos < start_pos
185
+ yield ( elem . event , str , elem . state )
186
+ byte_pos = end_pos
187
+ end
188
+
178
189
if lexer . respond_to? ( :scan ) # Ruby 2.7+
179
190
lexer . scan . each do |elem |
180
- str = elem . tok
181
191
next if allow_last_error and /meets end of file|unexpected end-of-input/ =~ elem . message
182
- next if ( [ elem . pos [ 0 ] , elem . pos [ 1 ] + str . bytesize ] <=> pos ) <= 0
183
-
184
- str . each_line do |line |
185
- if line . end_with? ( "\n " )
186
- pos [ 0 ] += 1
187
- pos [ 1 ] = 0
188
- else
189
- pos [ 1 ] += line . bytesize
190
- end
191
- end
192
-
193
- yield ( elem . event , str , elem . state )
192
+ on_scan . call ( elem )
194
193
end
195
194
else
196
- lexer . parse . each do |elem |
197
- yield ( elem . event , elem . tok , elem . state )
195
+ lexer . parse . sort_by ( & :pos ) . each do |elem |
196
+ on_scan . call ( elem )
198
197
end
199
198
end
199
+ # yield uncolorable DATA section
200
+ yield ( nil , inner_code . byteslice ( byte_pos ...inner_code . bytesize ) , nil ) if byte_pos < inner_code . bytesize
200
201
end
201
202
ensure
202
203
$VERBOSE = verbose
0 commit comments