Permalink
Switch branches/tags
Nothing to show
Find file
Fetching contributors…
Cannot retrieve contributors at this time
executable file 2878 lines (2348 sloc) 86.2 KB
# encoding: US-ASCII
require "minitest/autorun"
require "ruby_lexer"
require "ruby_parser"
class TestRubyLexer < Minitest::Test
attr_accessor :processor, :lex, :parser_class, :lex_state
alias :lexer :lex # lets me copy/paste code from parser
alias :lexer= :lex=
def setup
self.lex_state = :expr_beg
setup_lexer_class RubyParser.latest.class
end
def setup_lexer input, exp_sexp = nil
setup_new_parser
lex.ss = RPStringScanner.new(input)
lex.lex_state = self.lex_state
end
def setup_new_parser
self.processor = parser_class.new
self.lex = processor.lexer
end
def setup_lexer_class parser_class
self.parser_class = parser_class
setup_new_parser
setup_lexer "blah blah"
end
def assert_lex input, exp_sexp, *args, &b
setup_lexer input
assert_parse input, exp_sexp if exp_sexp
b.call if b
args.each_slice(5) do |token, value, state, paren, brace|
assert_next_lexeme token, value, state, paren, brace
end
refute_lexeme
end
def assert_lex3 input, exp_sexp, *args, &block
args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
assert_lex(input, exp_sexp, *args, &block)
end
def ruby18
RubyParser::V18 === lexer.parser
end
def refute_lex input, *args # TODO: re-sort
args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
assert_raises RubyParser::SyntaxError do
assert_lex(input, nil, *args)
end
end
def assert_lex_fname name, type, end_state = :expr_arg # TODO: swap name/type
assert_lex3("def #{name} ",
nil,
:kDEF, "def", :expr_fname,
type, name, end_state)
end
def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
adv = @lex.next_token
assert adv, "no more tokens"
act_token, act_value = adv
msg = message {
act = [act_token, act_value, @lex.lex_state,
@lex.paren_nest, @lex.brace_nest]
exp = [token, value, state, paren, brace]
"#{exp.inspect} vs #{act.inspect}"
}
act_value = act_value.first if Array === act_value
assert_equal token, act_token, msg
case value
when Float then
assert_in_epsilon value, act_value, 0.001, msg
when NilClass then
assert_nil act_value, msg
else
assert_equal value, act_value, msg
end
assert_equal state, @lex.lex_state, msg if state
assert_equal paren, @lex.paren_nest, msg if paren
assert_equal brace, @lex.brace_nest, msg if brace
end
def assert_parse input, exp_sexp
assert_equal exp_sexp, processor.class.new.parse(input)
end
def assert_read_escape expected, input
@lex.ss.string = input
assert_equal expected, @lex.read_escape, input
end
def assert_read_escape_bad input # TODO: rename refute_read_escape
@lex.ss.string = input
assert_raises RubyParser::SyntaxError do
@lex.read_escape
end
end
def refute_lexeme
x = y = @lex.next_token
refute x, "not empty: #{y.inspect}"
end
## Utility Methods:
def emulate_string_interpolation
lex_strterm = lexer.lex_strterm
string_nest = lexer.string_nest
brace_nest = lexer.brace_nest
lexer.string_nest = 0
lexer.brace_nest = 0
lexer.cond.push false
lexer.cmdarg.push false
lexer.lex_strterm = nil
lexer.lex_state = :expr_beg
yield
lexer.lex_state = :expr_endarg
assert_next_lexeme :tRCURLY, "}", :expr_endarg, 0
lexer.lex_strterm = lex_strterm
lexer.lex_state = :expr_beg
lexer.string_nest = string_nest
lexer.brace_nest = brace_nest
lexer.cond.lexpop
lexer.cmdarg.lexpop
end
## Tests:
def test_next_token
assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
assert_nil @lex.next_token
end
def test_unicode_ident
s = "@\u1088\u1077\u1093\u1072"
assert_lex3(s.dup, nil, :tIVAR, s.dup, :expr_end)
end
def test_read_escape
assert_read_escape "\\", "\\"
assert_read_escape "\n", "n"
assert_read_escape "\t", "t"
assert_read_escape "\r", "r"
assert_read_escape "\f", "f"
assert_read_escape "\13", "v"
assert_read_escape "\0", "0"
assert_read_escape "\07", "a"
assert_read_escape "\007", "a"
assert_read_escape "\033", "e"
assert_read_escape "\377", "377"
assert_read_escape "\377", "xff"
assert_read_escape "\010", "b"
assert_read_escape " ", "s"
assert_read_escape "q", "q" # plain vanilla escape
assert_read_escape "8", "8" # ugh... mri... WHY?!?
assert_read_escape "9", "9" # ugh... mri... WHY?!?
assert_read_escape "$", "444" # ugh
end
def test_read_escape_c
assert_read_escape "\030", "C-x"
assert_read_escape "\030", "cx"
assert_read_escape "\230", 'C-\M-x'
assert_read_escape "\230", 'c\M-x'
assert_read_escape "\177", "C-?"
assert_read_escape "\177", "c?"
end
def test_read_escape_errors
assert_read_escape_bad ""
assert_read_escape_bad "M"
assert_read_escape_bad "M-"
assert_read_escape_bad "Mx"
assert_read_escape_bad "Cx"
assert_read_escape_bad "C"
assert_read_escape_bad "C-"
assert_read_escape_bad "c"
end
def test_read_escape_m
assert_read_escape "\370", "M-x"
assert_read_escape "\230", 'M-\C-x'
assert_read_escape "\230", 'M-\cx'
end
def test_yylex_ambiguous_uminus
assert_lex3("m -3",
nil,
:tIDENTIFIER, "m", :expr_cmdarg,
:tUMINUS_NUM, "-", :expr_beg,
:tINTEGER, 3, :expr_end)
# TODO: verify warning
end
def test_yylex_ambiguous_uplus
assert_lex3("m +3",
nil,
:tIDENTIFIER, "m", :expr_cmdarg,
:tINTEGER, 3, :expr_end)
# TODO: verify warning
end
def test_yylex_and
assert_lex3("&", nil, :tAMPER, "&", :expr_beg)
end
def test_yylex_and2
assert_lex3("&&", nil, :tANDOP, "&&", :expr_beg)
end
def test_yylex_and2_equals
assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
end
def test_yylex_and_dot
setup_lexer_class RubyParser::V23
assert_lex3("&.", nil, :tLONELY, "&.", :expr_dot)
end
def test_yylex_and_dot_call
setup_lexer_class RubyParser::V23
assert_lex3("x&.y", nil,
:tIDENTIFIER, "x", :expr_cmdarg,
:tLONELY, "&.", :expr_dot,
:tIDENTIFIER, "y")
end
def test_yylex_and_dot_call_newline
setup_lexer_class Ruby23Parser
assert_lex3("x\n&.y", nil,
:tIDENTIFIER, "x", :expr_cmdarg,
:tLONELY, "&.", :expr_dot,
:tIDENTIFIER, "y")
end
def test_yylex_and_arg
self.lex_state = :expr_arg
assert_lex3(" &y",
nil,
:tAMPER, "&", :expr_beg,
:tIDENTIFIER, "y", :expr_arg)
end
def test_yylex_and_equals
assert_lex3("&=", nil, :tOP_ASGN, "&", :expr_beg)
end
def test_yylex_and_expr
self.lex_state = :expr_arg
assert_lex3("x & y",
nil,
:tIDENTIFIER, "x", :expr_cmdarg,
:tAMPER2, "&", :expr_beg,
:tIDENTIFIER, "y", :expr_arg)
end
def test_yylex_and_meth
assert_lex_fname "&", :tAMPER2
end
def test_yylex_assoc
assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
end
def test_yylex_label__18
setup_lexer_class RubyParser::V18
assert_lex3("{a:",
nil,
:tLBRACE, "{", :expr_beg,
:tIDENTIFIER, "a", :expr_arg,
:tSYMBEG, ":", :expr_fname)
end
def test_yylex_label_in_params__18
setup_lexer_class RubyParser::V18
assert_lex3("foo(a:",
nil,
:tIDENTIFIER, "foo", :expr_cmdarg,
:tLPAREN2, "(", :expr_beg,
:tIDENTIFIER, "a", :expr_cmdarg,
:tSYMBEG, ":", :expr_fname)
end
def test_yylex_label__19
setup_lexer_class RubyParser::V19
assert_lex3("{a:",
nil,
:tLBRACE, "{", :expr_beg,
:tLABEL, "a", :expr_labelarg)
end
def test_yylex_label_in_params__19
setup_lexer_class RubyParser::V19
assert_lex3("foo(a:",
nil,
:tIDENTIFIER, "foo", :expr_cmdarg,
:tLPAREN2, "(", :expr_beg,
:tLABEL, "a", :expr_labelarg)
end
def test_yylex_paren_string_parens_interpolated
setup_lexer('%((#{b}#{d}))',
s(:dstr,
"(",
s(:evstr, s(:call, nil, :b)),
s(:evstr, s(:call, nil, :d)),
s(:str, ")")))
assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
emulate_string_interpolation do
assert_next_lexeme :tIDENTIFIER, "b", :expr_arg, 0, 0
end
assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
emulate_string_interpolation do
assert_next_lexeme :tIDENTIFIER, "d", :expr_arg, 0, 0
end
assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
refute_lexeme
end
def test_yylex_paren_string_interpolated_regexp
setup_lexer('%( #{(/abcd/)} )',
s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
emulate_string_interpolation do
assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
end
assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
refute_lexeme
end
def test_yylex_not_at_defn__20
setup_lexer_class RubyParser::V20
assert_lex("def +@; end",
s(:defn, :+@, s(:args), s(:nil)),
:kDEF, "def", :expr_fname, 0, 0,
:tUPLUS, "+@", :expr_arg, 0, 0,
:tSEMI, ";", :expr_beg, 0, 0,
:kEND, "end", :expr_end, 0, 0)
assert_lex("def !@; end",
s(:defn, :"!@", s(:args), s(:nil)),
:kDEF, "def", :expr_fname, 0, 0,
:tUBANG, "!@", :expr_arg, 0, 0,
:tSEMI, ";", :expr_beg, 0, 0,
:kEND, "end", :expr_end, 0, 0)
end
def test_yylex_not_at_ivar
assert_lex("!@ivar",
s(:call, s(:ivar, :@ivar), :"!"),
:tBANG, "!", :expr_beg, 0, 0,
:tIVAR, "@ivar", :expr_end, 0, 0)
end
def test_yylex_number_times_ident_times_return_number
assert_lex("1 * b * 3",
s(:call,
s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
:*, s(:lit, 3)),
:tINTEGER, 1, :expr_end, 0, 0,
:tSTAR2, "*", :expr_beg, 0, 0,
:tIDENTIFIER, "b", :expr_arg, 0, 0,
:tSTAR2, "*", :expr_beg, 0, 0,
:tINTEGER, 3, :expr_end, 0, 0)
assert_lex("1 * b *\n 3",
s(:call,
s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
:*, s(:lit, 3)),
:tINTEGER, 1, :expr_end, 0, 0,
:tSTAR2, "*", :expr_beg, 0, 0,
:tIDENTIFIER, "b", :expr_arg, 0, 0,
:tSTAR2, "*", :expr_beg, 0, 0,
:tINTEGER, 3, :expr_end, 0, 0)
end
def test_yylex_paren_string_parens_interpolated_regexp
setup_lexer('%((#{(/abcd/)}))',
s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
emulate_string_interpolation do
assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
end
assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
refute_lexeme
end
def test_yylex_method_parens_chevron
assert_lex("a()<<1",
s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
:tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
:tLPAREN2, "(", :expr_beg, 1, 0,
:tRPAREN, ")", :expr_endfn, 0, 0,
:tLSHFT, "<<" , :expr_beg, 0, 0,
:tINTEGER, 1, :expr_end, 0, 0)
end
def test_yylex_lambda_args__20
setup_lexer_class RubyParser::V20
assert_lex("-> (a) { }",
s(:iter, s(:call, nil, :lambda),
s(:args, :a)),
:tLAMBDA, nil, :expr_endfn, 0, 0,
:tLPAREN2, "(", :expr_beg, 1, 0,
:tIDENTIFIER, "a", :expr_arg, 1, 0,
:tRPAREN, ")", :expr_endfn, 0, 0,
:tLCURLY, "{", :expr_beg, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_lambda_args_opt__20
setup_lexer_class RubyParser::V20
assert_lex("-> (a=nil) { }",
s(:iter, s(:call, nil, :lambda),
s(:args, s(:lasgn, :a, s(:nil)))),
:tLAMBDA, nil, :expr_endfn, 0, 0,
:tLPAREN2, "(", :expr_beg, 1, 0,
:tIDENTIFIER, "a", :expr_arg, 1, 0,
:tEQL, "=", :expr_beg, 1, 0,
:kNIL, "nil", :expr_end, 1, 0,
:tRPAREN, ")", :expr_endfn, 0, 0,
:tLCURLY, "{", :expr_beg, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_lambda_hash__20
setup_lexer_class RubyParser::V20
assert_lex("-> (a={}) { }",
s(:iter, s(:call, nil, :lambda),
s(:args, s(:lasgn, :a, s(:hash)))),
:tLAMBDA, nil, :expr_endfn, 0, 0,
:tLPAREN2, "(", :expr_beg, 1, 0,
:tIDENTIFIER, "a", :expr_arg, 1, 0,
:tEQL, "=", :expr_beg, 1, 0,
:tLBRACE, "{", :expr_beg, 1, 1,
:tRCURLY, "}", :expr_endarg, 1, 0,
:tRPAREN, ")", :expr_endfn, 0, 0,
:tLCURLY, "{", :expr_beg, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_iter_array_curly
assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
s(:iter,
s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
s(:args, :c, :d)),
:tIDENTIFIER, "f", :expr_cmdarg, 0, 0,
:tSYMBOL, "a", :expr_end, 0, 0,
:tCOMMA, ",", :expr_beg, 0, 0,
:tLBRACK, "[", :expr_beg, 1, 0,
:tSYMBOL, "b", :expr_end, 1, 0,
:tRBRACK, "]", :expr_endarg, 0, 0,
:tLBRACE_ARG, "{", :expr_beg, 0, 1,
:tPIPE, "|", :expr_beg, 0, 1,
:tIDENTIFIER, "c", :expr_arg, 0, 1,
:tCOMMA, ",", :expr_beg, 0, 1,
:tIDENTIFIER, "d", :expr_arg, 0, 1,
:tPIPE, "|", :expr_beg, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_const_call_same_name
assert_lex("X = a { }; b { f :c }",
s(:block,
s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
s(:iter,
s(:call, nil, :b),
0,
s(:call, nil, :f, s(:lit, :c)))),
:tCONSTANT, "X", :expr_cmdarg, 0, 0,
:tEQL, "=", :expr_beg, 0, 0,
:tIDENTIFIER, "a", :expr_arg, 0, 0,
:tLCURLY, "{", :expr_beg, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0,
:tSEMI, ";", :expr_beg, 0, 0,
:tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
:tLCURLY, "{", :expr_beg, 0, 1,
:tIDENTIFIER, "f", :expr_cmdarg, 0, 1, # different
:tSYMBOL, "c", :expr_end, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
assert_lex("X = a { }; b { X :c }",
s(:block,
s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
s(:iter,
s(:call, nil, :b),
0,
s(:call, nil, :X, s(:lit, :c)))),
:tCONSTANT, "X", :expr_cmdarg, 0, 0,
:tEQL, "=", :expr_beg, 0, 0,
:tIDENTIFIER, "a", :expr_arg, 0, 0,
:tLCURLY, "{", :expr_beg, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0,
:tSEMI, ";", :expr_beg, 0, 0,
:tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
:tLCURLY, "{", :expr_beg, 0, 1,
:tCONSTANT, "X", :expr_cmdarg, 0, 1, # same
:tSYMBOL, "c", :expr_end, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_lasgn_call_same_name
assert_lex("a = b.c :d => 1",
s(:lasgn, :a,
s(:call, s(:call, nil, :b), :c,
s(:hash, s(:lit, :d), s(:lit, 1)))),
:tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
:tEQL, "=", :expr_beg, 0, 0,
:tIDENTIFIER, "b", :expr_arg, 0, 0,
:tDOT, ".", :expr_dot, 0, 0,
:tIDENTIFIER, "c", :expr_arg, 0, 0, # different
:tSYMBOL, "d", :expr_end, 0, 0,
:tASSOC, "=>", :expr_beg, 0, 0,
:tINTEGER, 1, :expr_end, 0, 0)
assert_lex("a = b.a :d => 1",
s(:lasgn, :a,
s(:call, s(:call, nil, :b), :a,
s(:hash, s(:lit, :d), s(:lit, 1)))),
:tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
:tEQL, "=", :expr_beg, 0, 0,
:tIDENTIFIER, "b", :expr_arg, 0, 0,
:tDOT, ".", :expr_dot, 0, 0,
:tIDENTIFIER, "a", :expr_arg, 0, 0, # same as lvar
:tSYMBOL, "d", :expr_end, 0, 0,
:tASSOC, "=>", :expr_beg, 0, 0,
:tINTEGER, 1, :expr_end, 0, 0)
end
def test_yylex_back_ref
assert_lex3("[$&, $`, $', $+]",
nil,
:tLBRACK, "[", :expr_beg,
:tBACK_REF, :&, :expr_end, :tCOMMA, ",", :expr_beg,
:tBACK_REF, :"`", :expr_end, :tCOMMA, ",", :expr_beg,
:tBACK_REF, :"'", :expr_end, :tCOMMA, ",", :expr_beg,
:tBACK_REF, :+, :expr_end,
:tRBRACK, "]", :expr_endarg)
end
def test_yylex_backslash
assert_lex3("1 \\\n+ 2",
nil,
:tINTEGER, 1, :expr_end,
:tPLUS, "+", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_backslash_bad
refute_lex("1 \\ + 2", :tINTEGER, 1)
end
def test_yylex_backtick
assert_lex3("`ls`",
nil,
:tXSTRING_BEG, "`", :expr_beg,
:tSTRING_CONTENT, "ls", :expr_beg,
:tSTRING_END, "`", :expr_end)
end
def test_yylex_backtick_cmdarg
self.lex_state = :expr_dot
# \n ensures expr_cmd (TODO: why?)
assert_lex3("\n`", nil, :tBACK_REF2, "`", :expr_cmdarg)
end
def test_yylex_backtick_dot
self.lex_state = :expr_dot
assert_lex3("a.`(3)",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tDOT, ".", :expr_dot,
:tBACK_REF2, "`", :expr_arg,
:tLPAREN2, "(", :expr_beg,
:tINTEGER, 3, :expr_end,
:tRPAREN, ")", :expr_endfn)
end
def test_yylex_backtick_method
self.lex_state = :expr_fname
assert_lex3("`",
nil,
:tBACK_REF2, "`", :expr_end)
end
def test_yylex_bad_char
refute_lex(" \010 ")
end
def test_yylex_bang
assert_lex3("!", nil, :tBANG, "!", :expr_beg)
end
def test_yylex_bang_equals
assert_lex3("!=", nil, :tNEQ, "!=", :expr_beg)
end
def test_yylex_bang_tilde
assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
end
def test_yylex_carat
assert_lex3("^", nil, :tCARET, "^", :expr_beg)
end
def test_yylex_carat_equals
assert_lex3("^=", nil, :tOP_ASGN, "^", :expr_beg)
end
def test_yylex_colon2
assert_lex3("A::B",
nil,
:tCONSTANT, "A", :expr_cmdarg,
:tCOLON2, "::", :expr_dot,
:tCONSTANT, "B", :expr_arg)
end
def test_yylex_colon2_argh
assert_lex3("module X::Y\n c\nend",
nil,
:kMODULE, "module", :expr_value,
:tCONSTANT, "X", :expr_arg,
:tCOLON2, "::", :expr_dot,
:tCONSTANT, "Y", :expr_arg,
:tNL, nil, :expr_beg,
:tIDENTIFIER, "c", :expr_cmdarg,
:tNL, nil, :expr_beg,
:kEND, "end", :expr_end)
end
def test_yylex_colon3
assert_lex3("::Array",
nil,
:tCOLON3, "::", :expr_beg,
:tCONSTANT, "Array", :expr_arg)
end
def test_yylex_comma
assert_lex3(",", nil, :tCOMMA, ",", :expr_beg)
end
def test_yylex_comment
assert_lex3("1 # one\n# two\n2",
nil,
:tINTEGER, 1, :expr_end,
:tNL, nil, :expr_beg,
:tINTEGER, 2, :expr_end)
assert_equal "# one\n# two\n", @lex.comments
end
def test_yylex_comment_begin
assert_lex3("=begin\nblah\nblah\n=end\n42",
nil,
:tINTEGER, 42, :expr_end)
assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
end
def test_yylex_comment_begin_bad
refute_lex("=begin\nblah\nblah\n")
assert_equal "", @lex.comments
end
def test_yylex_comment_begin_not_comment
assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
nil,
:tIDENTIFIER, "beginfoo", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tINTEGER, 5, :expr_end,
:tNL, nil, :expr_beg,
:tIDENTIFIER, "p", :expr_cmdarg,
:tIDENTIFIER, "x", :expr_arg,
:tEQL, "=", :expr_beg,
:tIDENTIFIER, "beginfoo", :expr_arg)
end
def test_yylex_comment_begin_space
assert_lex3("=begin blah\nblah\n=end\n", nil)
assert_equal "=begin blah\nblah\n=end\n", @lex.comments
end
def test_yylex_comment_end_space_and_text
assert_lex3("=begin blah\nblah\n=end blab\n", nil)
assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
end
def test_yylex_comment_eos
assert_lex3("# comment", nil)
end
def test_yylex_constant
assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", :expr_cmdarg)
end
def test_yylex_constant_semi
assert_lex3("ArgumentError;",
nil,
:tCONSTANT, "ArgumentError", :expr_cmdarg,
:tSEMI, ";", :expr_beg)
end
def test_yylex_cvar
assert_lex3("@@blah", nil, :tCVAR, "@@blah", :expr_end)
end
def test_yylex_cvar_bad
assert_raises RubyParser::SyntaxError do
assert_lex3("@@1", nil)
end
end
def test_yylex_def_bad_name
self.lex_state = :expr_fname
refute_lex("def [ ", :kDEF, "def")
end
def test_yylex_div
assert_lex3("a / 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tDIVIDE, "/", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_div_equals
assert_lex3("a /= 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tOP_ASGN, "/", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_do
assert_lex3("x do 42 end",
nil,
:tIDENTIFIER, "x", :expr_cmdarg,
:kDO, "do", :expr_beg,
:tINTEGER, 42, :expr_end,
:kEND, "end", :expr_end)
end
def test_yylex_do_block
self.lex_state = :expr_endarg
assert_lex3("x.y do 42 end",
nil,
:tIDENTIFIER, "x", :expr_end,
:tDOT, ".", :expr_dot,
:tIDENTIFIER, "y", :expr_arg,
:kDO_BLOCK, "do", :expr_beg,
:tINTEGER, 42, :expr_end,
:kEND, "end", :expr_end) do
@lex.cmdarg.push true
end
end
def test_yylex_do_block2
self.lex_state = :expr_endarg
assert_lex3("do 42 end",
nil,
:kDO_BLOCK, "do", :expr_beg,
:tINTEGER, 42, :expr_end,
:kEND, "end", :expr_end)
end
def test_yylex_is_your_spacebar_broken?
assert_lex3(":a!=:b",
nil,
:tSYMBOL, "a", :expr_end,
:tNEQ, "!=", :expr_beg,
:tSYMBOL, "b", :expr_end)
end
def test_yylex_do_cond
assert_lex3("x do 42 end",
nil,
:tIDENTIFIER, "x", :expr_cmdarg,
:kDO_COND, "do", :expr_beg,
:tINTEGER, 42, :expr_end,
:kEND, "end", :expr_end) do
@lex.cond.push true
end
end
def test_yylex_dollar_bad
e = refute_lex("$%")
assert_includes(e.message, "is not allowed as a global variable name")
end
def test_yylex_dollar_eos
assert_lex3("$", nil, "$", "$", :expr_end) # FIX: wtf is this?!?
end
def test_yylex_dot # HINT message sends
assert_lex3(".", nil, :tDOT, ".", :expr_dot)
end
def test_yylex_dot2
assert_lex3("..", nil, :tDOT2, "..", :expr_beg)
end
def test_yylex_dot3
assert_lex3("...", nil, :tDOT3, "...", :expr_beg)
end
def test_yylex_equals
# FIX: this sucks
assert_lex3("=", nil, :tEQL, "=", :expr_beg)
end
def test_yylex_equals2
assert_lex3("==", nil, :tEQ, "==", :expr_beg)
end
def test_yylex_equals3
assert_lex3("===", nil, :tEQQ, "===", :expr_beg)
end
def test_yylex_equals_tilde
assert_lex3("=~", nil, :tMATCH, "=~", :expr_beg)
end
def test_yylex_float
assert_lex3("1.0", nil, :tFLOAT, 1.0, :expr_end)
end
def test_yylex_float_bad_no_underscores
refute_lex "1__0.0"
end
def test_yylex_float_bad_no_zero_leading
refute_lex ".0"
end
def test_yylex_float_bad_trailing_underscore
refute_lex "123_.0"
end
def test_yylex_float_call
assert_lex3("1.0.to_s",
nil,
:tFLOAT, 1.0, :expr_end,
:tDOT, ".", :expr_dot,
:tIDENTIFIER, "to_s", :expr_arg)
end
def test_yylex_float_dot_E
assert_lex3("1.0E10",
nil,
:tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_dot_E_neg
assert_lex3("-1.0E10",
nil,
:tUMINUS_NUM, "-", :expr_beg,
:tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_dot_e
assert_lex3("1.0e10",
nil,
:tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_dot_e_neg
assert_lex3("-1.0e10",
nil,
:tUMINUS_NUM, "-", :expr_beg,
:tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_e
assert_lex3("1e10",
nil,
:tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_e_bad_double_e
assert_lex3("1e2e3",
nil,
:tFLOAT, 100, :expr_end,
:tIDENTIFIER, "e3", :expr_end)
end
def test_yylex_float_if_modifier
assert_lex3("1e2if",
nil,
:tFLOAT, 100, :expr_end,
:kIF_MOD, "if", :expr_beg)
end
def test_yylex_float_e_bad_trailing_underscore
refute_lex "123_e10"
end
def test_yylex_float_e_minus
assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, :expr_end)
end
def test_yylex_float_e_neg
assert_lex3("-1e10",
nil,
:tUMINUS_NUM, "-", :expr_beg,
:tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_e_neg_minus
assert_lex3("-1e-10",
nil,
:tUMINUS_NUM, "-", :expr_beg,
:tFLOAT, 1.0e-10, :expr_end)
end
def test_yylex_float_e_neg_plus
assert_lex3("-1e+10",
nil,
:tUMINUS_NUM, "-", :expr_beg,
:tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_e_plus
assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, :expr_end)
end
def test_yylex_float_e_zero
assert_lex3("0e0", nil, :tFLOAT, 0.0, :expr_end)
end
def test_yylex_float_neg
assert_lex3("-1.0",
nil,
:tUMINUS_NUM, "-", :expr_beg,
:tFLOAT, 1.0, :expr_end)
end
def test_yylex_ge
assert_lex3("a >= 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tGEQ, ">=", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_global
assert_lex3("$blah", nil, :tGVAR, "$blah", :expr_end)
end
def test_yylex_global_backref
self.lex_state = :expr_fname
assert_lex3("$`", nil, :tGVAR, "$`", :expr_end)
end
def test_yylex_global_dash_nothing
assert_lex3("$- ", nil, :tGVAR, "$-", :expr_end)
end
def test_yylex_global_dash_something
assert_lex3("$-x", nil, :tGVAR, "$-x", :expr_end)
end
def test_yylex_global_number
self.lex_state = :expr_fname
assert_lex3("$1", nil, :tGVAR, "$1", :expr_end)
end
def test_yylex_global_number_big
self.lex_state = :expr_fname
assert_lex3("$1234", nil, :tGVAR, "$1234", :expr_end)
end
def test_yylex_global_other
assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
nil,
:tLBRACK, "[", :expr_beg,
:tGVAR, "$~", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$*", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$$", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$?", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$!", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$@", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$/", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$\\", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$;", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$,", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$.", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$=", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$:", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$<", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$>", :expr_end, :tCOMMA, ",", :expr_beg,
:tGVAR, "$\"", :expr_end,
:tRBRACK, "]", :expr_endarg)
end
def test_yylex_global_underscore
assert_lex3("$_", nil, :tGVAR, "$_", :expr_end)
end
def test_yylex_global_wierd
assert_lex3("$__blah", nil, :tGVAR, "$__blah", :expr_end)
end
def test_yylex_global_zero
assert_lex3("$0", nil, :tGVAR, "$0", :expr_end)
end
def test_yylex_gt
assert_lex3("a > 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tGT, ">", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_heredoc_backtick
assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tXSTRING_BEG, "`", :expr_beg,
:tSTRING_CONTENT, " blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_double
assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, " blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_double_dash
assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, " blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_double_squiggly
setup_lexer_class Ruby23Parser
assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
# mri handles tabs in a pretty specific way:
# https://github.com/ruby/ruby/blob/trunk/parse.y#L5925
def test_yylex_heredoc_double_squiggly_with_tab_indentation_remaining
setup_lexer_class Ruby23Parser
assert_lex3("a = <<~\"EOF\"\n blah blah\n \tblah blah\n EOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah blah\n\tblah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_double_squiggly_with_tab_indentation_removed
setup_lexer_class Ruby23Parser
assert_lex3("a = <<~\"EOF\"\n blah blah\n\t blah blah\n EOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah blah\n blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_double_eos
refute_lex("a = <<\"EOF\"\nblah",
:tIDENTIFIER, "a",
:tEQL, "=",
:tSTRING_BEG, "\"")
end
def test_yylex_heredoc_double_eos_nl
refute_lex("a = <<\"EOF\"\nblah\n",
:tIDENTIFIER, "a",
:tEQL, "=",
:tSTRING_BEG, "\"")
end
def test_yylex_heredoc_double_interp
assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "#x a ", :expr_beg,
:tSTRING_DVAR, "\#@", :expr_beg,
:tSTRING_CONTENT, "@a b ", :expr_beg, # HUH?
:tSTRING_DVAR, "\#$", :expr_beg,
:tSTRING_CONTENT, "$b c ", :expr_beg, # HUH?
:tSTRING_DBEG, "\#{", :expr_beg,
:tSTRING_CONTENT, "3} \n", :expr_beg, # HUH?
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_empty
assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_DBEG, "\#{", :expr_beg,
:tSTRING_CONTENT, "x}\nblah2\n", :expr_beg,
:tSTRING_END, "", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_none
assert_lex3("a = <<EOF\nblah\nblah\nEOF\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_none_bad_eos
refute_lex("a = <<EOF",
:tIDENTIFIER, "a",
:tEQL, "=",
:tSTRING_BEG, "\"")
end
def test_yylex_heredoc_none_dash
assert_lex3("a = <<-EOF\nblah\nblah\n EOF\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_none_squiggly
setup_lexer_class Ruby23Parser
assert_lex3("a = <<~EOF\n blah\n blah\n EOF\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_single
assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, " blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_single_bad_eos_body
refute_lex("a = <<'EOF'\nblah",
:tIDENTIFIER, "a",
:tEQL, "=",
:tSTRING_BEG, "\"")
end
def test_yylex_heredoc_single_bad_eos_empty
refute_lex("a = <<''\n",
:tIDENTIFIER, "a",
:tEQL, "=",
:tSTRING_BEG, "\"")
end
def test_yylex_heredoc_single_bad_eos_term
refute_lex("a = <<'EOF",
:tIDENTIFIER, "a",
:tEQL, "=",
:tSTRING_BEG, "\"")
end
def test_yylex_heredoc_single_bad_eos_term_nl
refute_lex("a = <<'EOF\ns = 'blah blah'",
:tIDENTIFIER, "a",
:tEQL, "=",
:tSTRING_BEG, "\"")
end
def test_yylex_heredoc_single_dash
assert_lex3("a = <<-'EOF'\n blah blah\n EOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, " blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_heredoc_single_squiggly
setup_lexer_class Ruby23Parser
assert_lex3("a = <<~'EOF'\n blah blah\n EOF\n\n",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah blah\n", :expr_beg,
:tSTRING_END, "EOF", :expr_end,
:tNL, nil, :expr_beg)
end
def test_yylex_identifier
assert_lex3("identifier",
nil,
:tIDENTIFIER, "identifier", :expr_cmdarg)
end
def test_yylex_identifier_bang
assert_lex3("identifier!",
nil,
:tFID, "identifier!", :expr_cmdarg)
end
def test_yylex_identifier_cmp
assert_lex_fname "<=>", :tCMP
end
def test_yylex_identifier_def__18
setup_lexer_class RubyParser::V18
assert_lex_fname "identifier", :tIDENTIFIER, :expr_end
end
def test_yylex_identifier_def__1920
setup_lexer_class RubyParser::V19
assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
end
def test_yylex_identifier_eh
assert_lex3("identifier?", nil, :tFID, "identifier?", :expr_cmdarg)
end
def test_yylex_identifier_equals_arrow
assert_lex3(":blah==>",
nil,
:tSYMBOL, "blah=", :expr_end,
:tASSOC, "=>", :expr_beg)
end
def test_yylex_identifier_equals3
assert_lex3(":a===b",
nil,
:tSYMBOL, "a", :expr_end,
:tEQQ, "===", :expr_beg,
:tIDENTIFIER, "b", :expr_arg)
end
def test_yylex_identifier_equals_equals_arrow
assert_lex3(":a==>b",
nil,
:tSYMBOL, "a=", :expr_end,
:tASSOC, "=>", :expr_beg,
:tIDENTIFIER, "b", :expr_arg)
end
def test_yylex_identifier_equals_caret
assert_lex_fname "^", :tCARET
end
def test_yylex_identifier_equals_def__18
setup_lexer_class RubyParser::V18
assert_lex_fname "identifier=", :tIDENTIFIER, :expr_end
end
def test_yylex_identifier_equals_def__1920
setup_lexer_class RubyParser::V19
assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
end
def test_yylex_identifier_equals_def2
assert_lex_fname "==", :tEQ
end
def test_yylex_identifier_equals_expr
self.lex_state = :expr_dot
assert_lex3("y = arg",
nil,
:tIDENTIFIER, "y", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tIDENTIFIER, "arg", :expr_arg)
end
def test_yylex_identifier_equals_or
assert_lex_fname "|", :tPIPE
end
def test_yylex_identifier_equals_slash
assert_lex_fname "/", :tDIVIDE
end
def test_yylex_identifier_equals_tilde
self.lex_state = :expr_fname # can only set via parser's defs
assert_lex3("identifier=~",
nil,
:tIDENTIFIER, "identifier", :expr_endfn,
:tMATCH, "=~", :expr_beg)
end
def test_yylex_identifier_gt
assert_lex_fname ">", :tGT
end
def test_yylex_identifier_le
assert_lex_fname "<=", :tLEQ
end
def test_yylex_identifier_lt
assert_lex_fname "<", :tLT
end
def test_yylex_identifier_tilde
assert_lex_fname "~", :tTILDE
end
def test_yylex_index
assert_lex_fname "[]", :tAREF
end
def test_yylex_index_equals
assert_lex_fname "[]=", :tASET
end
def test_yylex_integer
assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_integer_bin
assert_lex3("0b101010", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_integer_bin_bad_none
refute_lex "0b "
end
def test_yylex_integer_bin_bad_underscores
refute_lex "0b10__01"
end
def test_yylex_integer_dec
assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_integer_dec_bad_underscores
refute_lex "42__24"
end
def test_yylex_integer_dec_d
assert_lex3("0d42", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_integer_dec_d_bad_none
refute_lex "0d"
end
def test_yylex_integer_dec_d_bad_underscores
refute_lex "0d42__24"
end
def test_yylex_integer_if_modifier
assert_lex3("123if",
nil,
:tINTEGER, 123, :expr_end,
:kIF_MOD, "if", :expr_beg)
end
def test_yylex_question_eh_a__18
setup_lexer_class RubyParser::V18
assert_lex3("?a", nil, :tINTEGER, 97, :expr_end)
end
def test_yylex_question_eh_a__19
setup_lexer_class RubyParser::V19
assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
end
def test_yylex_question_eh_escape_M_escape_C__18
setup_lexer_class RubyParser::V18
assert_lex3("?\\M-\\C-a", nil, :tINTEGER, 129, :expr_end)
end
def test_yylex_question_eh_escape_M_escape_C__19
setup_lexer_class RubyParser::V19
assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
end
def test_yylex_integer_hex
assert_lex3 "0x2a", nil, :tINTEGER, 42, :expr_end
end
def test_yylex_integer_hex_bad_none
refute_lex "0x "
end
def test_yylex_integer_hex_bad_underscores
refute_lex "0xab__cd"
end
def test_yylex_integer_oct
assert_lex3("052", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_integer_oct_bad_range
refute_lex "08"
end
def test_yylex_integer_oct_bad_range2
refute_lex "08"
end
def test_yylex_integer_oct_bad_underscores
refute_lex "01__23"
end
def test_yylex_integer_oct_O
assert_lex3 "0O52", nil, :tINTEGER, 42, :expr_end
end
def test_yylex_integer_oct_O_bad_range
refute_lex "0O8"
end
def test_yylex_integer_oct_O_bad_underscores
refute_lex "0O1__23"
end
def test_yylex_integer_oct_O_not_bad_none
assert_lex3 "0O ", nil, :tINTEGER, 0, :expr_end
end
def test_yylex_integer_oct_o
assert_lex3 "0o52", nil, :tINTEGER, 42, :expr_end
end
def test_yylex_integer_oct_o_bad_range
refute_lex "0o8"
end
def test_yylex_integer_oct_o_bad_underscores
refute_lex "0o1__23"
end
def test_yylex_integer_oct_o_not_bad_none
assert_lex3 "0o ", nil, :tINTEGER, 0, :expr_end
end
def test_yylex_integer_trailing
assert_lex3("1.to_s",
nil,
:tINTEGER, 1, :expr_end,
:tDOT, ".", :expr_dot,
:tIDENTIFIER, "to_s", :expr_arg)
end
def test_yylex_integer_underscore
assert_lex3("4_2", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_integer_underscore_bad
refute_lex "4__2"
end
def test_yylex_integer_zero
assert_lex3 "0", nil, :tINTEGER, 0, :expr_end
end
def test_yylex_ivar
assert_lex3("@blah", nil, :tIVAR, "@blah", :expr_end)
end
def test_yylex_ivar_bad
refute_lex "@1"
end
def test_yylex_ivar_bad_0_length
refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", :expr_end
end
def test_yylex_keyword_expr
self.lex_state = :expr_endarg
assert_lex3("if", nil, :kIF_MOD, "if", :expr_beg)
end
def test_yylex_lt
assert_lex3("<", nil, :tLT, "<", :expr_beg)
end
def test_yylex_lt2
assert_lex3("a << b",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tLSHFT, "<<", :expr_beg,
:tIDENTIFIER, "b", :expr_arg)
end
def test_yylex_lt2_equals
assert_lex3("a <<= b",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tOP_ASGN, "<<", :expr_beg,
:tIDENTIFIER, "b", :expr_arg)
end
def test_yylex_lt_equals
assert_lex3("<=", nil, :tLEQ, "<=", :expr_beg)
end
def test_yylex_minus
assert_lex3("1 - 2",
nil,
:tINTEGER, 1, :expr_end,
:tMINUS, "-", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_minus_equals
assert_lex3("-=", nil, :tOP_ASGN, "-", :expr_beg)
end
def test_yylex_minus_method
self.lex_state = :expr_fname
assert_lex3("-", nil, :tMINUS, "-", :expr_arg)
end
def test_yylex_minus_unary_method
self.lex_state = :expr_fname
assert_lex3("-@", nil, :tUMINUS, "-@", :expr_arg)
end
def test_yylex_minus_unary_number
assert_lex3("-42",
nil,
:tUMINUS_NUM, "-", :expr_beg,
:tINTEGER, 42, :expr_end)
end
def test_yylex_nth_ref
assert_lex3("[$1, $2, $3, $4, $5, $6, $7, $8, $9]",
nil,
:tLBRACK, "[", :expr_beg,
:tNTH_REF, 1, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 2, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 3, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 4, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 5, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 6, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 7, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 8, :expr_end, :tCOMMA, ",", :expr_beg,
:tNTH_REF, 9, :expr_end,
:tRBRACK, "]", :expr_endarg)
end
def test_yylex_open_bracket
assert_lex3("(", nil, :tLPAREN, "(", :expr_beg)
end
def test_yylex_open_bracket_cmdarg
self.lex_state = :expr_cmdarg
assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
end
def test_yylex_open_bracket_exprarg__18
setup_lexer_class RubyParser::V18
self.lex_state = :expr_arg
assert_lex3(" (", nil, :tLPAREN2, "(", :expr_beg)
end
def test_yylex_open_bracket_exprarg__19
setup_lexer_class RubyParser::V19
self.lex_state = :expr_arg
assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
end
def test_yylex_open_curly_bracket
assert_lex3("{", nil, :tLBRACE, "{", :expr_beg)
end
def test_yylex_open_curly_bracket_arg
self.lex_state = :expr_arg
assert_lex3("m { 3 }",
nil,
:tIDENTIFIER, "m", :expr_cmdarg,
:tLCURLY, "{", :expr_beg,
:tINTEGER, 3, :expr_end,
:tRCURLY, "}", :expr_endarg)
end
def test_yylex_open_curly_bracket_block
self.lex_state = :expr_endarg # seen m(3)
assert_lex3("{ 4 }",
nil,
:tLBRACE_ARG, "{", :expr_beg,
:tINTEGER, 4, :expr_end,
:tRCURLY, "}", :expr_endarg)
end
def test_yylex_open_square_bracket_arg
self.lex_state = :expr_arg
assert_lex3("m [ 3 ]",
nil,
:tIDENTIFIER, "m", :expr_cmdarg,
:tLBRACK, "[", :expr_beg,
:tINTEGER, 3, :expr_end,
:tRBRACK, "]", :expr_endarg)
end
def test_yylex_open_square_bracket_ary
assert_lex3("[1, 2, 3]",
nil,
:tLBRACK, "[", :expr_beg,
:tINTEGER, 1, :expr_end, :tCOMMA, ",", :expr_beg,
:tINTEGER, 2, :expr_end, :tCOMMA, ",", :expr_beg,
:tINTEGER, 3, :expr_end,
:tRBRACK, "]", :expr_endarg)
end
def test_yylex_open_square_bracket_meth
assert_lex3("m[3]",
nil,
:tIDENTIFIER, "m", :expr_cmdarg,
:tLBRACK2, "[", :expr_beg,
:tINTEGER, 3, :expr_end,
:tRBRACK, "]", :expr_endarg)
end
def test_yylex_or
assert_lex3("|", nil, :tPIPE, "|", :expr_beg)
end
def test_yylex_or2
assert_lex3("||", nil, :tOROP, "||", :expr_beg)
end
def test_yylex_or2_equals
assert_lex3("||=", nil, :tOP_ASGN, "||", :expr_beg)
end
def test_yylex_or_equals
assert_lex3("|=", nil, :tOP_ASGN, "|", :expr_beg)
end
def test_yylex_percent
assert_lex3("a % 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tPERCENT, "%", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_percent_equals
assert_lex3("a %= 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tOP_ASGN, "%", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_plus
assert_lex3("1 + 1", # TODO lex_state?
nil,
:tINTEGER, 1, :expr_end,
:tPLUS, "+", :expr_beg,
:tINTEGER, 1, :expr_end)
end
def test_yylex_plus_equals
assert_lex3("+=", nil, :tOP_ASGN, "+", :expr_beg)
end
def test_yylex_plus_method
self.lex_state = :expr_fname
assert_lex3("+", nil, :tPLUS, "+", :expr_arg)
end
def test_yylex_plus_unary_method
self.lex_state = :expr_fname
assert_lex3("+@", nil, :tUPLUS, "+@", :expr_arg)
end
def test_yylex_not_unary_method
self.lex_state = :expr_fname
assert_lex3("!@", nil, :tUBANG, "!@", :expr_arg)
end
def test_yylex_numbers
assert_lex3("0b10", nil, :tINTEGER, 2, :expr_end)
assert_lex3("0B10", nil, :tINTEGER, 2, :expr_end)
assert_lex3("0d10", nil, :tINTEGER, 10, :expr_end)
assert_lex3("0D10", nil, :tINTEGER, 10, :expr_end)
assert_lex3("0x10", nil, :tINTEGER, 16, :expr_end)
assert_lex3("0X10", nil, :tINTEGER, 16, :expr_end)
assert_lex3("0o10", nil, :tINTEGER, 8, :expr_end)
assert_lex3("0O10", nil, :tINTEGER, 8, :expr_end)
assert_lex3("0o", nil, :tINTEGER, 0, :expr_end)
assert_lex3("0O", nil, :tINTEGER, 0, :expr_end)
assert_lex3("0", nil, :tINTEGER, 0, :expr_end)
refute_lex "0x"
refute_lex "0X"
refute_lex "0b"
refute_lex "0B"
refute_lex "0d"
refute_lex "0D"
refute_lex "08"
refute_lex "09"
refute_lex "0o8"
refute_lex "0o9"
refute_lex "0O8"
refute_lex "0O9"
refute_lex "1_e1"
refute_lex "1_.1"
refute_lex "1__1"
end
def test_yylex_plus_unary_number
assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_question__18
setup_lexer_class RubyParser::V18
assert_lex3("?*", nil, :tINTEGER, 42, :expr_end)
end
def test_yylex_question__19
setup_lexer_class RubyParser::V19
assert_lex3("?*", nil, :tSTRING, "*", :expr_end)
end
def test_yylex_question_bad_eos
refute_lex "?"
end
def test_yylex_question_ws
assert_lex3("? ", nil, :tEH, "?", :expr_value)
assert_lex3("?\n", nil, :tEH, "?", :expr_value)
assert_lex3("?\t", nil, :tEH, "?", :expr_value)
assert_lex3("?\v", nil, :tEH, "?", :expr_value)
assert_lex3("?\r", nil, :tEH, "?", :expr_value)
assert_lex3("?\f", nil, :tEH, "?", :expr_value)
end
def test_yylex_question_ws_backslashed__18
setup_lexer_class RubyParser::V18
assert_lex3("?\\ ", nil, :tINTEGER, 32, :expr_end)
assert_lex3("?\\n", nil, :tINTEGER, 10, :expr_end)
assert_lex3("?\\t", nil, :tINTEGER, 9, :expr_end)
assert_lex3("?\\v", nil, :tINTEGER, 11, :expr_end)
assert_lex3("?\\r", nil, :tINTEGER, 13, :expr_end)
assert_lex3("?\\f", nil, :tINTEGER, 12, :expr_end)
end
def test_yylex_question_ws_backslashed__19
setup_lexer_class RubyParser::V19
assert_lex3("?\\ ", nil, :tSTRING, " ", :expr_end)
assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
assert_lex3("?\\t", nil, :tSTRING, "\t", :expr_end)
assert_lex3("?\\v", nil, :tSTRING, "\v", :expr_end)
assert_lex3("?\\r", nil, :tSTRING, "\r", :expr_end)
assert_lex3("?\\f", nil, :tSTRING, "\f", :expr_end)
end
def test_yylex_rbracket
assert_lex3("]", nil, :tRBRACK, "]", :expr_endarg)
end
def test_yylex_rcurly
assert_lex3("}", nil, :tRCURLY, "}", :expr_endarg)
end
def test_yylex_regexp
assert_lex3("/regexp/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regexp", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_ambiguous
assert_lex3("method /regexp/",
nil,
:tIDENTIFIER, "method", :expr_cmdarg,
:tREGEXP_BEG, "/", :expr_cmdarg,
:tSTRING_CONTENT, "regexp", :expr_cmdarg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_bad
refute_lex("/.*/xyz",
:tREGEXP_BEG, "/",
:tSTRING_CONTENT, ".*")
end
def test_yylex_regexp_escape_C
assert_lex3("/regex\\C-x/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\C-x", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_C_M
assert_lex3("/regex\\C-\\M-x/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_C_M_craaaazy
assert_lex3("/regex\\C-\\\n\\M-x/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_C_bad_dash
refute_lex '/regex\\Cx/', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_C_bad_dash_eos
refute_lex '/regex\\C-/', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_C_bad_dash_eos2
refute_lex '/regex\\C-', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_C_bad_eos
refute_lex '/regex\\C/', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_C_bad_eos2
refute_lex '/regex\\c', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_M
assert_lex3("/regex\\M-x/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\M-x", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_M_C
assert_lex3("/regex\\M-\\C-x/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\M-\\C-x", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_M_bad_dash
refute_lex '/regex\\Mx/', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_M_bad_dash_eos
refute_lex '/regex\\M-/', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_M_bad_dash_eos2
refute_lex '/regex\\M-', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_M_bad_eos
refute_lex '/regex\\M/', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_backslash_slash
assert_lex3("/\\//",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "\\/", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_backslash_terminator
assert_lex3("%r%blah\\%blah%",
nil,
:tREGEXP_BEG, "%r\000", :expr_beg,
:tSTRING_CONTENT, "blah\\%blah", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_backslash_terminator_meta1
assert_lex3("%r{blah\\}blah}",
nil,
:tREGEXP_BEG, "%r{", :expr_beg, # FIX ?!?
:tSTRING_CONTENT, "blah\\}blah", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_backslash_terminator_meta2
assert_lex3("%r/blah\\/blah/",
nil,
:tREGEXP_BEG, "%r\000", :expr_beg,
:tSTRING_CONTENT, "blah\\/blah", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_backslash_terminator_meta3
assert_lex3("%r/blah\\%blah/",
nil,
:tREGEXP_BEG, "%r\000", :expr_beg,
:tSTRING_CONTENT, "blah\\%blah", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_bad_eos
refute_lex '/regex\\', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_bs
assert_lex3("/regex\\\\regex/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\\\regex", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_c
assert_lex3("/regex\\cxxx/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\cxxx", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_c_backslash
assert_lex3("/regex\\c\\n/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\c\\n", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_chars
assert_lex3("/re\\tge\\nxp/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "re\\tge\\nxp", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_double_backslash
regexp = '/[\\/\\\\]$/'
assert_lex3(regexp.dup,
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "[\\/\\\\]$", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_hex
assert_lex3("/regex\\x61xp/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\x61xp", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_hex_bad
refute_lex '/regex\\xzxp/', :tREGEXP_BEG, "/"
end
def test_yylex_regexp_escape_hex_one
assert_lex3("/^[\\xd\\xa]{2}/on",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "^[\\xd\\xa]{2}", :expr_beg,
:tREGEXP_END, "on", :expr_end)
end
def test_yylex_regexp_escape_oct1
assert_lex3("/regex\\0xp/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\0xp", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_oct2
assert_lex3("/regex\\07xp/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\07xp", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_oct3
assert_lex3("/regex\\10142/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regex\\10142", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_escape_return
assert_lex3("/regex\\\nregex/",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, "regexregex", :expr_beg,
:tREGEXP_END, "", :expr_end)
end
def test_yylex_regexp_nm
assert_lex3("/.*/nm",
nil,
:tREGEXP_BEG, "/", :expr_beg,
:tSTRING_CONTENT, ".*", :expr_beg,
:tREGEXP_END, "nm", :expr_end)
end
def test_yylex_rparen
assert_lex3(")", nil, :tRPAREN, ")", :expr_endfn)
end
def test_yylex_rshft
assert_lex3("a >> 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tRSHFT, ">>", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_rshft_equals
assert_lex3("a >>= 2",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tOP_ASGN, ">>", :expr_beg,
:tINTEGER, 2, :expr_end)
end
def test_yylex_star
assert_lex3("a * ",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tSTAR2, "*", :expr_beg)
end
def test_yylex_star2
assert_lex3("a ** ",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tPOW, "**", :expr_beg)
end
def test_yylex_star2_equals
assert_lex3("a **= ",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tOP_ASGN, "**", :expr_beg)
end
def test_yylex_star_arg
self.lex_state = :expr_arg
assert_lex3(" *a",
nil,
:tSTAR, "*", :expr_beg,
:tIDENTIFIER, "a", :expr_arg)
end
def test_yylex_star_arg_beg
self.lex_state = :expr_beg
assert_lex3("*a",
nil,
:tSTAR, "*", :expr_beg,
:tIDENTIFIER, "a", :expr_arg)
end
def test_yylex_star_arg_beg_fname
self.lex_state = :expr_fname
assert_lex3("*a",
nil,
:tSTAR2, "*", :expr_arg,
:tIDENTIFIER, "a", :expr_arg)
end
def test_yylex_star_arg_beg_fname2
self.lex_state = :expr_fname
assert_lex3("*a",
nil,
:tSTAR2, "*", :expr_arg,
:tIDENTIFIER, "a", :expr_arg)
end
def test_yylex_star_equals
assert_lex3("a *= ",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tOP_ASGN, "*", :expr_beg)
end
def test_yylex_string_bad_eos
refute_lex('%', :tSTRING_BEG, '%')
end
def test_yylex_string_bad_eos_quote
refute_lex('%{nest', :tSTRING_BEG, '%}')
end
def test_yylex_string_double
assert_lex3("\"string\"", nil, :tSTRING, "string", :expr_end)
end
def test_yylex_string_double_escape_C
assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", :expr_end)
end
def test_yylex_string_double_escape_C_backslash
assert_lex3("\"\\C-\\\\\"",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "\034", :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_escape_C_escape
assert_lex3("\"\\C-\\M-a\"",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "\201", :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_escape_C_question
assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", :expr_end)
end
def test_yylex_string_utf8_simple
chr = [0x3024].pack("U")
assert_lex3('"\u{3024}"',
s(:str, chr),
:tSTRING, chr, :expr_end)
end
def test_yylex_string_utf8_complex
chr = [0x3024].pack("U")
assert_lex3('"#@a\u{3024}"',
s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
:tSTRING_BEG, '"', :expr_beg,
:tSTRING_DVAR, nil, :expr_beg,
:tSTRING_CONTENT, "@a"+chr, :expr_beg,
:tSTRING_END, '"', :expr_end)
end
def test_yylex_string_double_escape_M
chr = "\341"
chr.force_encoding("UTF-8") if RubyLexer::HAS_ENC
assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
end
def test_why_does_ruby_hate_me?
assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
nil,
:tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, :expr_end)
end
def test_yylex_string_double_escape_M_backslash
assert_lex3("\"\\M-\\\\\"",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "\334", :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_escape_M_escape
assert_lex3("\"\\M-\\C-a\"",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "\201", :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_escape_bs1
assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", :expr_end)
end
def test_yylex_string_double_escape_bs2
assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", :expr_end)
end
def test_yylex_string_double_escape_c
assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", :expr_end)
end
def test_yylex_string_double_escape_c_backslash
assert_lex3("\"\\c\\\"",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "\034", :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_escape_c_escape
assert_lex3("\"\\c\\M-a\"",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "\201", :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_escape_c_question
assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", :expr_end)
end
def test_yylex_string_double_escape_chars
assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", :expr_end)
end
def test_yylex_string_double_escape_hex
assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", :expr_end)
end
def test_yylex_string_double_escape_octal
assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", :expr_end)
end
def test_yylex_string_double_escape_octal_fucked
assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", :expr_end)
end
def test_yylex_string_double_interp
assert_lex3("\"blah #x a \#@a b \#$b c \#{3} # \"",
nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, "blah #x a ", :expr_beg,
:tSTRING_DVAR, nil, :expr_beg,
:tSTRING_CONTENT, "@a b ", :expr_beg,
:tSTRING_DVAR, nil, :expr_beg,
:tSTRING_CONTENT, "$b c ", :expr_beg,
:tSTRING_DBEG, nil, :expr_beg,
:tSTRING_CONTENT, "3} # ", :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_pound_dollar_bad
skip if ruby18
assert_lex3('"#$%"', nil,
:tSTRING_BEG, "\"", :expr_beg,
:tSTRING_CONTENT, '#$%', :expr_beg,
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_string_double_nested_curlies
assert_lex3("%{nest{one{two}one}nest}",
nil,
:tSTRING_BEG, "%}", :expr_beg,
:tSTRING_CONTENT, "nest{one{two}one}nest", :expr_beg,
:tSTRING_END, "}", :expr_end)
end
def test_yylex_string_double_no_interp
assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", :expr_end)
assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", :expr_end)
end
def test_yylex_string_escape_x_single
assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", :expr_end)
end
def test_yylex_string_pct_i
assert_lex3("%i[s1 s2\ns3]",
nil,
:tQSYMBOLS_BEG, "%i[", :expr_beg,
:tSTRING_CONTENT, "s1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s3", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_pct_I
assert_lex3("%I[s1 s2\ns3]",
nil,
:tSYMBOLS_BEG, "%I[", :expr_beg,
:tSTRING_CONTENT, "s1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s3", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_pct_i_extra_space
assert_lex3("%i[ s1 s2\ns3 ]",
nil,
:tQSYMBOLS_BEG, "%i[", :expr_beg,
:tSTRING_CONTENT, "s1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s3", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_pct_I_extra_space
assert_lex3("%I[ s1 s2\ns3 ]",
nil,
:tSYMBOLS_BEG, "%I[", :expr_beg,
:tSTRING_CONTENT, "s1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s3", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_pct_q
assert_lex3("%q[s1 s2]",
nil,
:tSTRING_BEG, "%q[", :expr_beg,
:tSTRING_CONTENT, "s1 s2", :expr_beg,
:tSTRING_END, "]", :expr_end)
end
def test_yylex_string_pct_Q
assert_lex3("%Q[s1 s2]",
nil,
:tSTRING_BEG, "%Q[", :expr_beg,
:tSTRING_CONTENT, "s1 s2", :expr_beg,
:tSTRING_END, "]", :expr_end)
end
def test_yylex_string_pct_W
assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
nil,
:tWORDS_BEG, "%W[", :expr_beg,
:tSTRING_CONTENT, "s1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s3", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_pct_W_bs_nl
assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
nil,
:tWORDS_BEG, "%W[", :expr_beg,
:tSTRING_CONTENT, "s1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "\ns2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_pct_angle
assert_lex3("%<blah>",
nil,
:tSTRING_BEG, "%>", :expr_beg,
:tSTRING_CONTENT, "blah", :expr_beg,
:tSTRING_END, ">", :expr_end)
end
def test_yylex_string_pct_other
assert_lex3("%%blah%",
nil,
:tSTRING_BEG, "%%", :expr_beg,
:tSTRING_CONTENT, "blah", :expr_beg,
:tSTRING_END, "%", :expr_end)
end
def test_yylex_string_pct_w
refute_lex("%w[s1 s2 ",
:tQWORDS_BEG, "%w[",
:tSTRING_CONTENT, "s1",
:tSPACE, nil,
:tSTRING_CONTENT, "s2",
:tSPACE, nil)
end
def test_yylex_string_pct_w_bs_nl
assert_lex3("%w[s1 \\\ns2]",
nil,
:tQWORDS_BEG, "%w[", :expr_beg,
:tSTRING_CONTENT, "s1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "\ns2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_pct_w_bs_sp
assert_lex3("%w[s\\ 1 s\\ 2]",
nil,
:tQWORDS_BEG, "%w[", :expr_beg,
:tSTRING_CONTENT, "s 1", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_CONTENT, "s 2", :expr_beg,
:tSPACE, nil, :expr_beg,
:tSTRING_END, nil, :expr_end)
end
def test_yylex_string_single
assert_lex3("'string'", nil, :tSTRING, "string", :expr_end)
end
def test_yylex_string_single_escape_chars
assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", :expr_end)
end
def test_yylex_string_single_nl
assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", :expr_end)
end
def test_yylex_symbol
assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
end
def test_yylex_symbol_zero_byte__18
setup_lexer_class RubyParser::V18
refute_lex(":\"symbol\0\"", :tSYMBEG, ":")
end
def test_yylex_symbol_zero_byte
assert_lex(":\"symbol\0\"", nil,
:tSYMBOL, "symbol\0", :expr_end)
end
def test_yylex_symbol_double
assert_lex3(":\"symbol\"",
nil,
:tSYMBOL, "symbol", :expr_end)
end
def test_yylex_symbol_double_interp
assert_lex3(':"symbol#{1+1}"',
nil,
:tSYMBEG, ":", :expr_fname,
:tSTRING_CONTENT, "symbol", :expr_fname,
:tSTRING_DBEG, nil, :expr_fname,
:tSTRING_CONTENT, "1+1}", :expr_fname, # HUH? this is BS
:tSTRING_END, "\"", :expr_end)
end
def test_yylex_symbol_single
assert_lex3(":'symbol'",
nil,
:tSYMBOL, "symbol", :expr_end)
end
def test_yylex_symbol_single_noninterp
assert_lex3(':\'symbol#{1+1}\'',
nil,
:tSYMBOL, 'symbol#{1+1}', :expr_end)
end
def test_yylex_ternary1
assert_lex3("a ? b : c",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEH, "?", :expr_value,
:tIDENTIFIER, "b", :expr_arg,
:tCOLON, ":", :expr_beg,
:tIDENTIFIER, "c", :expr_arg)
assert_lex3("a ?bb : c", # GAH! MATZ!!!
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEH, "?", :expr_beg,
:tIDENTIFIER, "bb", :expr_arg,
:tCOLON, ":", :expr_beg,
:tIDENTIFIER, "c", :expr_arg)
assert_lex3("42 ?",
nil,
:tINTEGER, 42, :expr_end,
:tEH, "?", :expr_value)
end
def test_yylex_tilde
assert_lex3("~", nil, :tTILDE, "~", :expr_beg)
end
def test_yylex_tilde_unary
self.lex_state = :expr_fname
assert_lex3("~@", nil, :tTILDE, "~", :expr_arg)
end
def test_yylex_uminus
assert_lex3("-blah",
nil,
:tUMINUS, "-", :expr_beg,
:tIDENTIFIER, "blah", :expr_arg)
end
def test_yylex_underscore
assert_lex3("_var", nil, :tIDENTIFIER, "_var", :expr_cmdarg)
end
def test_yylex_underscore_end
assert_lex3("__END__\n",
nil,
RubyLexer::EOF, RubyLexer::EOF, nil)
end
def test_yylex_uplus
assert_lex3("+blah",
nil,
:tUPLUS, "+", :expr_beg,
:tIDENTIFIER, "blah", :expr_arg)
end
def test_zbug_float_in_decl
assert_lex3("def initialize(u = 0.0, s = 0.0",
nil,
:kDEF, "def", :expr_fname,
:tIDENTIFIER, "initialize", :expr_endfn,
:tLPAREN2, "(", :expr_beg,
:tIDENTIFIER, "u", :expr_arg,
:tEQL, "=", :expr_beg,
:tFLOAT, 0.0, :expr_end,
:tCOMMA, ",", :expr_beg,
:tIDENTIFIER, "s", :expr_arg,
:tEQL, "=", :expr_beg,
:tFLOAT, 0.0, :expr_end)
end
def test_zbug_id_equals
assert_lex3("a = 0.0",
nil,
:tIDENTIFIER, "a", :expr_cmdarg,
:tEQL, "=", :expr_beg,
:tFLOAT, 0.0, :expr_end)
end
def test_zbug_no_spaces_in_decl
assert_lex3("def initialize(u=0.0,s=0.0",
nil,
:kDEF, "def", :expr_fname,
:tIDENTIFIER, "initialize", :expr_endfn,
:tLPAREN2, "(", :expr_beg,
:tIDENTIFIER, "u", :expr_arg,
:tEQL, "=", :expr_beg,
:tFLOAT, 0.0, :expr_end,
:tCOMMA, ",", :expr_beg,
:tIDENTIFIER, "s", :expr_arg,
:tEQL, "=", :expr_beg,
:tFLOAT, 0.0, :expr_end)
end
def test_pct_w_backslashes
["\t", "\n", "\r", "\v", "\f"].each do |char|
next if !RubyLexer::HAS_ENC and char == "\v"
assert_lex("%w[foo#{char}bar]",
s(:array, s(:str, "foo"), s(:str, "bar")),
:tQWORDS_BEG, "%w[", :expr_beg, 0, 0,
:tSTRING_CONTENT, "foo", :expr_beg, 0, 0,
:tSPACE, nil, :expr_beg, 0, 0,
:tSTRING_CONTENT, "bar", :expr_beg, 0, 0,
:tSPACE, nil, :expr_beg, 0, 0,
:tSTRING_END, nil, :expr_end, 0, 0)
end
end
def test_yylex_sym_quoted
assert_lex(":'a'",
s(:lit, :a),
:tSYMBOL, "a", :expr_end, 0, 0)
end
def test_yylex_hash_colon
assert_lex("{a:1}",
s(:hash, s(:lit, :a), s(:lit, 1)),
:tLBRACE, "{", :expr_beg, 0, 1,
:tLABEL, "a", :expr_labelarg, 0, 1,
:tINTEGER, 1, :expr_end, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_hash_colon_quoted_22
setup_lexer_class RubyParser::V22
assert_lex("{'a':1}",
s(:hash, s(:lit, :a), s(:lit, 1)),
:tLBRACE, "{", :expr_beg, 0, 1,
:tLABEL, "a", :expr_labelarg, 0, 1,
:tINTEGER, 1, :expr_end, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_hash_colon_quoted_symbol_22
setup_lexer_class RubyParser::V22
assert_lex("{'abc': :b}",
s(:hash, s(:lit, :abc), s(:lit, :b)),
:tLBRACE, "{", :expr_beg, 0, 1,
:tLABEL, "abc", :expr_labelarg, 0, 1,
:tSYMBOL, "b", :expr_end, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_yylex_hash_colon_double_quoted_symbol_22
setup_lexer_class RubyParser::V22
assert_lex('{"abc": :b}',
s(:hash, s(:lit, :abc), s(:lit, :b)),
:tLBRACE, "{", :expr_beg, 0, 1,
:tLABEL, "abc", :expr_labelarg, 0, 1,
:tSYMBOL, "b", :expr_end, 0, 1,
:tRCURLY, "}", :expr_endarg, 0, 0)
end
def test_ruby21_rational_literal
setup_lexer_class RubyParser::V21
assert_lex3("10r", nil, :tRATIONAL, Rational(10), :expr_end)
assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), :expr_end)
assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), :expr_end)
assert_lex3("0or", nil, :tRATIONAL, Rational(0), :expr_end)
assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), :expr_end)
assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), :expr_end)
assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), :expr_end)
assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), :expr_end)
assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), :expr_end)
assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), :expr_end)
assert_lex3("-10r", nil,
:tUMINUS_NUM, "-", :expr_beg,
:tRATIONAL, Rational(10), :expr_end)
end
def test_ruby21_imaginary_literal
setup_lexer_class RubyParser::V21
assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), :expr_end)
assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), :expr_end)
assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), :expr_end)
assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), :expr_end)
assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), :expr_end)
assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), :expr_end)
assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), :expr_end)
assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), :expr_end)
assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), :expr_end)
assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), :expr_end)
assert_lex3("-10i", nil,
:tUMINUS_NUM, "-", :expr_beg,
:tIMAGINARY, Complex(0, 10), :expr_end)
end
def test_ruby21_rational_imaginary_literal
setup_lexer_class RubyParser::V21
assert_lex3("1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), :expr_end)
assert_lex3("0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), :expr_end)
assert_lex3("0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), :expr_end)
assert_lex3("0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), :expr_end)
assert_lex3("0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), :expr_end)
assert_lex3("1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), :expr_end)
assert_lex3("15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), :expr_end)
assert_lex3("15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), :expr_end)
assert_lex3("1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), :expr_end)
assert_lex3("1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), :expr_end)
assert_lex3("-10ri", nil,
:tUMINUS_NUM, "-", :expr_beg,
:tIMAGINARY, Complex(0, Rational(10)), :expr_end)
end
def test_ruby21_imaginary_literal_with_succeeding_keyword
skip "Currently does not tokenize correctly"
setup_lexer_class RubyParser::V21
assert_lex3("1if", nil,
:tINTEGER, 1, :expr_end,
:kIF_MOD, "if", :expr_beg)
assert_lex3("1rif", nil,
:tRATIONAL, Rational(1), :expr_end,
:kIF_MOD, "if", :expr_beg)
assert_lex3("1.0if", nil,
:tFLOAT, 1.0, :expr_end,
:kIF_MOD, "if", :expr_beg)
assert_lex3("1.0rif", nil,
:tRATIONAL, Rational("1.0"), :expr_end,
:kIF_MOD, "if", :expr_beg)
flunk
end
end