Skip to content

Commit e90f88f

Browse files
committed
Move lex compat into its own file
1 parent 1c843d2 commit e90f88f

File tree

2 files changed

+52
-41
lines changed

2 files changed

+52
-41
lines changed

lib/yarp.rb

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,8 @@ def pretty_print(q)
303303
autoload :DesugarCompiler, "yarp/desugar_compiler"
304304
autoload :Dispatcher, "yarp/dispatcher"
305305
autoload :DSL, "yarp/dsl"
306+
autoload :LexCompat, "yarp/lex_compat"
307+
autoload :LexRipper, "yarp/lex_compat"
306308
autoload :MutationCompiler, "yarp/mutation_compiler"
307309
autoload :NodeInspector, "yarp/node_inspector"
308310
autoload :RipperCompat, "yarp/ripper_compat"
@@ -311,20 +313,33 @@ def pretty_print(q)
311313
autoload :Serialize, "yarp/serialize"
312314
autoload :Visitor, "yarp/visitor"
313315

314-
# Marking this as private so that consumers don't see it. It makes it a little
315-
# annoying for testing since you have to const_get it to access the methods,
316-
# but at least this way it's clear it's not meant for consumers.
316+
# Some of these constants are not meant to be exposed, so marking them as
317+
# private here.
317318
private_constant :Debug
319+
private_constant :LexCompat
320+
private_constant :LexRipper
321+
322+
# Returns an array of tokens that closely resembles that of the Ripper lexer.
323+
# The only difference is that since we don't keep track of lexer state in the
324+
# same way, it's going to always return the NONE state.
325+
def self.lex_compat(source, filepath = "")
326+
LexCompat.new(source, filepath).result
327+
end
328+
329+
# This lexes with the Ripper lex. It drops any space events but otherwise
330+
# returns the same tokens. Raises SyntaxError if the syntax in source is
331+
# invalid.
332+
def self.lex_ripper(source)
333+
LexRipper.new(source).result
334+
end
318335

319336
# Load the serialized AST using the source as a reference into a tree.
320337
def self.load(source, serialized)
321338
Serialize.load(source, serialized)
322339
end
323340
end
324341

325-
require_relative "yarp/lex_compat"
326342
require_relative "yarp/node"
327-
328343
require_relative "yarp/parse_result/comments"
329344
require_relative "yarp/parse_result/newlines"
330345

lib/yarp/lex_compat.rb

Lines changed: 32 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -794,48 +794,44 @@ def result
794794
end
795795
end
796796

797-
# The constant that wraps the behavior of the lexer to match Ripper's output
798-
# is an implementation detail, so we don't want it to be public.
799-
private_constant :LexCompat
800-
801-
# Returns an array of tokens that closely resembles that of the Ripper lexer.
802-
# The only difference is that since we don't keep track of lexer state in the
803-
# same way, it's going to always return the NONE state.
804-
def self.lex_compat(source, filepath = "")
805-
LexCompat.new(source, filepath).result
806-
end
797+
# This is a class that wraps the Ripper lexer to produce almost exactly the
798+
# same tokens.
799+
class LexRipper
800+
attr_reader :source
807801

808-
# This lexes with the Ripper lex. It drops any space events but otherwise
809-
# returns the same tokens. Raises SyntaxError if the syntax in source is
810-
# invalid.
811-
def self.lex_ripper(source)
812-
previous = []
813-
results = []
814-
815-
Ripper.lex(source, raise_errors: true).each do |token|
816-
case token[1]
817-
when :on_sp
818-
# skip
819-
when :on_tstring_content
820-
if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
821-
previous[2] << token[2]
822-
else
823-
results << token
824-
previous = token
825-
end
826-
when :on_words_sep
827-
if previous[1] == :on_words_sep
828-
previous[2] << token[2]
802+
def initialize(source)
803+
@source = source
804+
end
805+
806+
def result
807+
previous = []
808+
results = []
809+
810+
Ripper.lex(source, raise_errors: true).each do |token|
811+
case token[1]
812+
when :on_sp
813+
# skip
814+
when :on_tstring_content
815+
if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
816+
previous[2] << token[2]
817+
else
818+
results << token
819+
previous = token
820+
end
821+
when :on_words_sep
822+
if previous[1] == :on_words_sep
823+
previous[2] << token[2]
824+
else
825+
results << token
826+
previous = token
827+
end
829828
else
830829
results << token
831830
previous = token
832831
end
833-
else
834-
results << token
835-
previous = token
836832
end
837-
end
838833

839-
results
834+
results
835+
end
840836
end
841837
end

0 commit comments

Comments
 (0)