diff --git a/lib/parser/lexer.rl b/lib/parser/lexer.rl index a2368c584..9d5869c44 100644 --- a/lib/parser/lexer.rl +++ b/lib/parser/lexer.rl @@ -2027,7 +2027,7 @@ class Parser::Lexer end if version?(18, 19, 20) - emit(:tINTEGER, digits.to_i(@num_base)) + emit(:tINTEGER, digits.to_i(@num_base), @ts, @num_suffix_s) p = @num_suffix_s - 1 else @num_xfrm.call(digits.to_i(@num_base)) @@ -2047,7 +2047,7 @@ class Parser::Lexer :trailing_in_number, { :character => tok(@te - 1, @te) }, range(@te - 1, @te) else - emit(:tINTEGER, tok(@ts, @te - 1).to_i) + emit(:tINTEGER, tok(@ts, @te - 1).to_i, @ts, @te - 1) fhold; fbreak; end }; @@ -2059,7 +2059,7 @@ class Parser::Lexer :trailing_in_number, { :character => tok(@te - 1, @te) }, range(@te - 1, @te) else - emit(:tFLOAT, tok(@ts, @te - 1).to_f) + emit(:tFLOAT, tok(@ts, @te - 1).to_f, @ts, @te - 1) fhold; fbreak; end }; @@ -2072,7 +2072,7 @@ class Parser::Lexer digits = tok(@ts, @num_suffix_s) if version?(18, 19, 20) - emit(:tFLOAT, Float(digits)) + emit(:tFLOAT, Float(digits), @ts, @num_suffix_s) p = @num_suffix_s - 1 else @num_xfrm.call(digits) diff --git a/test/test_lexer.rb b/test/test_lexer.rb index f420a0e57..134b5ec07 100644 --- a/test/test_lexer.rb +++ b/test/test_lexer.rb @@ -70,8 +70,11 @@ def refute_escape(input) assert_equal :fatal, err.diagnostic.level end - def assert_lex_fname(name, type) - assert_scanned("def #{name} ", :kDEF, 'def', type, name) + def assert_lex_fname(name, type, range) + begin_pos, end_pos = range + assert_scanned("def #{name} ", + :kDEF, 'def', [0, 3], + type, name, [begin_pos + 4, end_pos + 4]) assert_equal :expr_endfn, @lex.state end @@ -84,11 +87,13 @@ def assert_scanned(input, *args) @lex.source_buffer = source_buffer until args.empty? do - token, value = args.shift(2) + token, value, (begin_pos, end_pos) = args.shift(3) - lex_token, (lex_value, *) = @lex.advance + lex_token, (lex_value, lex_range) = @lex.advance assert lex_token, 'no more tokens' assert_operator [lex_token, lex_value], :eql?, [token, value], input + assert_equal begin_pos, lex_range.begin_pos + assert_equal end_pos, lex_range.end_pos end lex_token, (lex_value, *) = @lex.advance @@ -172,151 +177,151 @@ def test_read_escape_unicode_bad__19 def test_ambiguous_uminus assert_scanned("m -3", - :tIDENTIFIER, "m", - :tUMINUS_NUM, "-", - :tINTEGER, 3) + :tIDENTIFIER, "m", [0, 1], + :tUMINUS_NUM, "-", [2, 3], + :tINTEGER, 3, [3, 4]) end def test_ambiguous_uplus assert_scanned("m +3", - :tIDENTIFIER, "m", - :tINTEGER, 3) + :tIDENTIFIER, "m", [0, 1], + :tINTEGER, 3, [3, 4]) end def test_and - assert_scanned "&", :tAMPER, "&" + assert_scanned "&", :tAMPER, "&", [0, 1] end def test_and2 @lex.state = :expr_end - assert_scanned "&&", :tANDOP, "&&" + assert_scanned "&&", :tANDOP, "&&", [0, 2] end def test_and2_equals @lex.state = :expr_end - assert_scanned "&&=", :tOP_ASGN, "&&" + assert_scanned "&&=", :tOP_ASGN, "&&", [0, 3] end def test_and_arg @lex.state = :expr_arg assert_scanned(" &y", - :tAMPER, "&", - :tIDENTIFIER, "y") + :tAMPER, "&", [1, 2], + :tIDENTIFIER, "y", [2, 3]) end def test_and_equals @lex.state = :expr_end - assert_scanned "&=", :tOP_ASGN, "&" + assert_scanned "&=", :tOP_ASGN, "&", [0, 2] end def test_and_expr @lex.state = :expr_arg assert_scanned("x & y", - :tIDENTIFIER, "x", - :tAMPER2, "&", - :tIDENTIFIER, "y") + :tIDENTIFIER, "x", [0, 1], + :tAMPER2, "&", [2, 3], + :tIDENTIFIER, "y", [4, 5]) end def test_and_meth - assert_lex_fname "&", :tAMPER2 + assert_lex_fname "&", :tAMPER2, [0, 1] end def test_and_dot @lex.state = :expr_cmdarg - assert_scanned "&.", :tANDDOT, "&." + assert_scanned "&.", :tANDDOT, "&.", [0, 2] end def test_assoc - assert_scanned "=>", :tASSOC, "=>" + assert_scanned "=>", :tASSOC, "=>", [0, 2] end def test_label__18 assert_scanned("{a:b", - :tLBRACE, "{", - :tIDENTIFIER, "a", - :tSYMBOL, "b") + :tLBRACE, "{", [0, 1], + :tIDENTIFIER, "a", [1, 2], + :tSYMBOL, "b", [2, 4]) end def test_label_in_params__18 assert_scanned("foo(a:b", - :tIDENTIFIER, "foo", - :tLPAREN2, "(", - :tIDENTIFIER, "a", - :tSYMBOL, "b") + :tIDENTIFIER, "foo", [0, 3], + :tLPAREN2, "(", [3, 4], + :tIDENTIFIER, "a", [4, 5], + :tSYMBOL, "b", [5, 7]) end def test_label__19 setup_lexer 19 assert_scanned("{a:b", - :tLBRACE, "{", - :tLABEL, "a", - :tIDENTIFIER, "b") + :tLBRACE, "{", [0, 1], + :tLABEL, "a", [1, 3], + :tIDENTIFIER, "b", [3, 4]) end def test_label_in_params__19 setup_lexer 19 assert_scanned("foo(a:b", - :tIDENTIFIER, "foo", - :tLPAREN2, "(", - :tLABEL, "a", - :tIDENTIFIER, "b") + :tIDENTIFIER, "foo", [0, 3], + :tLPAREN2, "(", [3, 4], + :tLABEL, "a", [4, 6], + :tIDENTIFIER, "b", [6, 7]) end def test_label_fid__19 setup_lexer 19 assert_scanned("{a?:true", - :tLBRACE, '{', - :tLABEL, 'a?', - :kTRUE, 'true') + :tLBRACE, '{', [0, 1], + :tLABEL, 'a?', [1, 4], + :kTRUE, 'true', [4, 8]) end def test_label__22 setup_lexer 22 assert_scanned("{'a':", - :tLBRACE, '{', - :tSTRING_BEG, "'", - :tSTRING_CONTENT, 'a', - :tLABEL_END, "'") + :tLBRACE, '{', [0, 1], + :tSTRING_BEG, "'", [1, 2], + :tSTRING_CONTENT, 'a', [2, 3], + :tLABEL_END, "'", [3, 5]) end def test_label_nested__22 setup_lexer 22 assert_scanned("{'a\":':", - :tLBRACE, '{', - :tSTRING_BEG, "'", - :tSTRING_CONTENT, 'a":', - :tLABEL_END, "'") + :tLBRACE, '{', [0, 1], + :tSTRING_BEG, "'", [1, 2], + :tSTRING_CONTENT, 'a":', [2, 5], + :tLABEL_END, "'", [5, 7]) end def test_label_colon2__22 setup_lexer 22 assert_scanned("{'a'::", - :tLBRACE, '{', - :tSTRING, "a", - :tCOLON2, '::') + :tLBRACE, '{', [0, 1], + :tSTRING, "a", [1, 4], + :tCOLON2, '::', [4, 6]) end def test_pct_string_colon__22 setup_lexer 22 assert_scanned("{%'a':", - :tLBRACE, '{', - :tSTRING_BEG, "%'", - :tSTRING_CONTENT, 'a', - :tSTRING_END, "'", - :tCOLON, ':') + :tLBRACE, '{', [0, 1], + :tSTRING_BEG, "%'", [1, 3], + :tSTRING_CONTENT, 'a', [3, 4], + :tSTRING_END, "'", [4, 5], + :tCOLON, ':', [5, 6]) end def test_command_start__19 @@ -328,9 +333,9 @@ def test_command_start__19 @lex.reset assert_scanned("#{keyword} a:b", - token, keyword, - :tIDENTIFIER, "a", - :tSYMBOL, "b") + token, keyword, [0, keyword.length], + :tIDENTIFIER, "a", [keyword.length + 1, keyword.length + 2], + :tSYMBOL, "b", [keyword.length + 2, keyword.length + 4]) end end @@ -342,44 +347,44 @@ def test_mod_not_command_start__19 @lex.state = :expr_end assert_scanned("#{keyword} a:b", - token, keyword, - :tLABEL, "a", - :tIDENTIFIER, "b") + token, keyword, [0, keyword.length], + :tLABEL, "a", [keyword.length + 1, keyword.length + 3], + :tIDENTIFIER, "b", [keyword.length + 3, keyword.length + 4]) end end def test_back_ref assert_scanned("[$&, $`, $', $+]", - :tLBRACK, "[", - :tBACK_REF, "$&", :tCOMMA, ",", - :tBACK_REF, "$`", :tCOMMA, ",", - :tBACK_REF, "$'", :tCOMMA, ",", - :tBACK_REF, "$+", - :tRBRACK, "]") + :tLBRACK, "[", [0, 1], + :tBACK_REF, "$&", [1, 3], :tCOMMA, ",", [3, 4], + :tBACK_REF, "$`", [5, 7], :tCOMMA, ",", [7, 8], + :tBACK_REF, "$'", [9, 11], :tCOMMA, ",", [11, 12], + :tBACK_REF, "$+", [13, 15], + :tRBRACK, "]", [15, 16]) end def test_backslash assert_scanned("1 \\\n+ 2", - :tINTEGER, 1, - :tPLUS, "+", - :tINTEGER, 2) + :tINTEGER, 1, [0, 1], + :tPLUS, "+", [4, 5], + :tINTEGER, 2, [6, 7]) end def test_backslash_bad refute_scanned("1 \\ + 2", - :tINTEGER, 1) + :tINTEGER, 1, [0, 1]) end def test_backtick assert_scanned("`ls`", - :tXSTRING_BEG, "`", - :tSTRING_CONTENT, "ls", - :tSTRING_END, "`") + :tXSTRING_BEG, "`", [0, 1], + :tSTRING_CONTENT, "ls", [1, 3], + :tSTRING_END, "`", [3, 4]) end def test_backtick_cmdarg @lex.state = :expr_dot - assert_scanned("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd + assert_scanned("\n`", :tBACK_REF2, "`", [1, 2]) # \n ensures expr_cmd assert_equal :expr_arg, @lex.state end @@ -387,17 +392,17 @@ def test_backtick_cmdarg def test_backtick_dot @lex.state = :expr_dot assert_scanned("a.`(3)", - :tIDENTIFIER, "a", - :tDOT, ".", - :tBACK_REF2, "`", - :tLPAREN2, "(", - :tINTEGER, 3, - :tRPAREN, ")") + :tIDENTIFIER, "a", [0, 1], + :tDOT, ".", [1, 2], + :tBACK_REF2, "`", [2, 3], + :tLPAREN2, "(", [3, 4], + :tINTEGER, 3, [4, 5], + :tRPAREN, ")", [5, 6]) end def test_backtick_method @lex.state = :expr_fname - assert_scanned("`", :tBACK_REF2, "`") + assert_scanned("`", :tBACK_REF2, "`", [0, 1]) assert_equal :expr_endfn, @lex.state end @@ -406,64 +411,64 @@ def test_bad_char end def test_bang - assert_scanned "!", :tBANG, "!" + assert_scanned "!", :tBANG, "!", [0, 1] end def test_bang_equals - assert_scanned "!=", :tNEQ, "!=" + assert_scanned "!=", :tNEQ, "!=", [0, 2] end def test_bang_tilde - assert_scanned "!~", :tNMATCH, "!~" + assert_scanned "!~", :tNMATCH, "!~", [0, 2] end def test_def_ubang setup_lexer(20) @lex.state = :expr_fname - assert_scanned '!@', :tBANG, '!@' + assert_scanned '!@', :tBANG, '!@', [0, 2] end def test_carat - assert_scanned "^", :tCARET, "^" + assert_scanned "^", :tCARET, "^", [0, 1] end def test_carat_equals - assert_scanned "^=", :tOP_ASGN, "^" + assert_scanned "^=", :tOP_ASGN, "^", [0, 2] end def test_colon2 assert_scanned("A::B", - :tCONSTANT, "A", - :tCOLON2, "::", - :tCONSTANT, "B") + :tCONSTANT, "A", [0, 1], + :tCOLON2, "::", [1, 3], + :tCONSTANT, "B", [3, 4]) @lex.state = :expr_arg assert_scanned("::Array", - :tCOLON2, "::", - :tCONSTANT, "Array") + :tCOLON2, "::", [0, 2], + :tCONSTANT, "Array", [2, 7]) end def test_colon3 assert_scanned("::Array", - :tCOLON3, "::", - :tCONSTANT, "Array") + :tCOLON3, "::", [0, 2], + :tCONSTANT, "Array", [2, 7]) @lex.state = :expr_arg assert_scanned(" ::Array", - :tCOLON3, "::", - :tCONSTANT, "Array") + :tCOLON3, "::", [1, 3], + :tCONSTANT, "Array", [3, 8]) end def test_comma - assert_scanned ",", :tCOMMA, "," + assert_scanned ",", :tCOMMA, ",", [0, 1] end def test_comment assert_scanned("1 # one\n# two\n2", - :tINTEGER, 1, - :tNL, nil, - :tINTEGER, 2) + :tINTEGER, 1, [0, 1], + :tNL, nil, [7, 8], + :tINTEGER, 2, [14, 15]) assert_equal 2, @lex.comments.length assert_equal '# one', @lex.comments[0].text @@ -472,13 +477,13 @@ def test_comment def test_comment_expr_beg assert_scanned("{#1\n}", - :tLBRACE, "{", - :tRCURLY, "}") + :tLBRACE, "{", [0, 1], + :tRCURLY, "}", [4, 5]) end def test_comment_begin assert_scanned("=begin\nblah\nblah\n=end\n42", - :tINTEGER, 42) + :tINTEGER, 42, [22, 24]) assert_equal 1, @lex.comments.length assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments[0].text end @@ -489,14 +494,14 @@ def test_comment_begin_bad def test_comment_begin_not_comment assert_scanned("beginfoo = 5\np x \\\n=beginfoo", - :tIDENTIFIER, "beginfoo", - :tEQL, "=", - :tINTEGER, 5, - :tNL, nil, - :tIDENTIFIER, "p", - :tIDENTIFIER, "x", - :tEQL, "=", - :tIDENTIFIER, "beginfoo") + :tIDENTIFIER, "beginfoo", [0, 8], + :tEQL, "=", [9, 10], + :tINTEGER, 5, [11, 12], + :tNL, nil, [12, 13], + :tIDENTIFIER, "p", [13, 14], + :tIDENTIFIER, "x", [15, 16], + :tEQL, "=", [19, 20], + :tIDENTIFIER, "beginfoo", [20, 28]) end def test_comment_begin_space @@ -519,17 +524,17 @@ def test_comment_eos def test_constant assert_scanned("ArgumentError", - :tCONSTANT, "ArgumentError") + :tCONSTANT, "ArgumentError", [0, 13]) end def test_constant_semi assert_scanned("ArgumentError;", - :tCONSTANT, "ArgumentError", - :tSEMI, ";") + :tCONSTANT, "ArgumentError", [0, 13], + :tSEMI, ";", [13, 14]) end def test_cvar - assert_scanned "@@blah", :tCVAR, "@@blah" + assert_scanned "@@blah", :tCVAR, "@@blah", [0, 6] end def test_cvar_bad @@ -538,85 +543,85 @@ def test_cvar_bad def test_div assert_scanned("a / 2", - :tIDENTIFIER, "a", - :tDIVIDE, "/", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tDIVIDE, "/", [2, 3], + :tINTEGER, 2, [4, 5]) end def test_div_equals assert_scanned("a /= 2", - :tIDENTIFIER, "a", - :tOP_ASGN, "/", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tOP_ASGN, "/", [2, 4], + :tINTEGER, 2, [5, 6]) end def test_do assert_scanned("x do 42 end", - :tIDENTIFIER, "x", - :kDO, "do", - :tINTEGER, 42, - :kEND, "end") + :tIDENTIFIER, "x", [0, 1], + :kDO, "do", [2, 4], + :tINTEGER, 42, [5, 7], + :kEND, "end", [8, 11]) end def test_do_cond @lex.cond.push(true) assert_scanned("x do 42 end", - :tIDENTIFIER, "x", - :kDO_COND, "do", - :tINTEGER, 42, - :kEND, "end") + :tIDENTIFIER, "x", [0, 1], + :kDO_COND, "do", [2, 4], + :tINTEGER, 42, [5, 7], + :kEND, "end", [8, 11]) end def test_do_block @lex.state = :expr_endarg assert_scanned("do 42 end", - :kDO_BLOCK, "do", - :tINTEGER, 42, - :kEND, "end") + :kDO_BLOCK, "do", [0, 2], + :tINTEGER, 42, [3, 5], + :kEND, "end", [6, 9]) end def test_do_cond @lex.cond.push true assert_scanned("x do 42 end", - :tIDENTIFIER, "x", - :kDO_COND, "do", - :tINTEGER, 42, - :kEND, "end") + :tIDENTIFIER, "x", [0, 1], + :kDO_COND, "do", [2, 4], + :tINTEGER, 42, [5, 7], + :kEND, "end", [8, 11]) end def test_dot - assert_scanned ".", :tDOT, "." + assert_scanned ".", :tDOT, ".", [0, 1] end def test_dot2 - assert_scanned "..", :tDOT2, ".." + assert_scanned "..", :tDOT2, "..", [0, 2] end def test_dot3 - assert_scanned "...", :tDOT3, "..." + assert_scanned "...", :tDOT3, "...", [0, 3] end def test_equals - assert_scanned "=", :tEQL, "=" + assert_scanned "=", :tEQL, "=", [0, 1] end def test_equals2 - assert_scanned "==", :tEQ, "==" + assert_scanned "==", :tEQ, "==", [0, 2] end def test_equals3 - assert_scanned "===", :tEQQ, "===" + assert_scanned "===", :tEQQ, "===", [0, 3] end def test_equals_tilde - assert_scanned "=~", :tMATCH, "=~" + assert_scanned "=~", :tMATCH, "=~", [0, 2] end def test_float - assert_scanned "1.0", :tFLOAT, 1.0 + assert_scanned "1.0", :tFLOAT, 1.0, [0, 3] end def test_float_bad_no_underscores @@ -633,33 +638,33 @@ def test_float_bad_trailing_underscore def test_float_call assert_scanned("1.0.to_s", - :tFLOAT, 1.0, - :tDOT, ".", - :tIDENTIFIER, "to_s") + :tFLOAT, 1.0, [0, 3], + :tDOT, ".", [3, 4], + :tIDENTIFIER, "to_s", [4, 8]) end def test_float_dot_E - assert_scanned "1.0E10", :tFLOAT, 1.0e10 + assert_scanned "1.0E10", :tFLOAT, 1.0e10, [0, 6] end def test_float_dot_E_neg assert_scanned("-1.0E10", - :tUMINUS_NUM, "-", - :tFLOAT, 1.0e10) + :tUMINUS_NUM, "-", [0, 1], + :tFLOAT, 1.0e10, [1, 7]) end def test_float_dot_e - assert_scanned "1.0e10", :tFLOAT, 1.0e10 + assert_scanned "1.0e10", :tFLOAT, 1.0e10, [0, 6] end def test_float_dot_e_neg assert_scanned("-1.0e10", - :tUMINUS_NUM, "-", - :tFLOAT, 1.0e10) + :tUMINUS_NUM, "-", [0, 1], + :tFLOAT, 1.0e10, [1, 7]) end def test_float_e - assert_scanned "1e10", :tFLOAT, 1e10 + assert_scanned "1e10", :tFLOAT, 1e10, [0, 4] end def test_float_e_bad_trailing_underscore @@ -667,33 +672,33 @@ def test_float_e_bad_trailing_underscore end def test_float_e_minus - assert_scanned "1e-10", :tFLOAT, 1e-10 + assert_scanned "1e-10", :tFLOAT, 1e-10, [0, 5] end def test_float_e_neg assert_scanned("-1e10", - :tUMINUS_NUM, "-", - :tFLOAT, 1e10) + :tUMINUS_NUM, "-", [0, 1], + :tFLOAT, 1e10, [1, 5]) end def test_float_e_neg_minus assert_scanned("-1e-10", - :tUMINUS_NUM, "-", - :tFLOAT, 1e-10) + :tUMINUS_NUM, "-", [0, 1], + :tFLOAT, 1e-10, [1, 6]) end def test_float_e_neg_plus assert_scanned("-1e+10", - :tUMINUS_NUM, "-", - :tFLOAT, 1e10) + :tUMINUS_NUM, "-", [0, 1], + :tFLOAT, 1e10, [1, 6]) end def test_float_e_plus - assert_scanned "1e+10", :tFLOAT, 1e10 + assert_scanned "1e+10", :tFLOAT, 1e10, [0, 5] end def test_float_e_zero - assert_scanned "0e0", :tFLOAT, 0e0 + assert_scanned "0e0", :tFLOAT, 0e0, [0, 3] end def test_float_e_nothing @@ -707,32 +712,32 @@ def test_float_e_nothing setup_lexer 21 assert_scanned("1end", - :tINTEGER, 1, - :kEND, 'end') + :tINTEGER, 1, [0, 1], + :kEND, 'end', [1, 4]) assert_scanned("1.1end", - :tFLOAT, 1.1, - :kEND, 'end') + :tFLOAT, 1.1, [0, 3], + :kEND, 'end', [3, 6]) end def test_float_neg assert_scanned("-1.0", - :tUMINUS_NUM, "-", - :tFLOAT, 1.0) + :tUMINUS_NUM, "-", [0, 1], + :tFLOAT, 1.0, [1, 4]) end def test_ge assert_scanned("a >= 2", - :tIDENTIFIER, "a", - :tGEQ, ">=", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tGEQ, ">=", [2, 4], + :tINTEGER, 2, [5, 6]) end def test_global - assert_scanned("$blah", :tGVAR, "$blah") + assert_scanned("$blah", :tGVAR, "$blah", [0, 5]) end def test_global_backref - assert_scanned("$`", :tBACK_REF, "$`") + assert_scanned("$`", :tBACK_REF, "$`", [0, 2]) end # This was removed in 2.1. @@ -741,327 +746,327 @@ def test_global_backref # end def test_global_dash_something - assert_scanned("$-x", :tGVAR, "$-x") + assert_scanned("$-x", :tGVAR, "$-x", [0, 3]) end def test_global_number - assert_scanned("$10", :tNTH_REF, 10) + assert_scanned("$10", :tNTH_REF, 10, [0, 3]) end def test_global_other assert_scanned("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]", - :tLBRACK, "[", - :tGVAR, "$~", :tCOMMA, ",", - :tGVAR, "$*", :tCOMMA, ",", - :tGVAR, "$$", :tCOMMA, ",", - :tGVAR, "$\?", :tCOMMA, ",", - :tGVAR, "$!", :tCOMMA, ",", - :tGVAR, "$@", :tCOMMA, ",", - :tGVAR, "$/", :tCOMMA, ",", - :tGVAR, "$\\", :tCOMMA, ",", - :tGVAR, "$;", :tCOMMA, ",", - :tGVAR, "$,", :tCOMMA, ",", - :tGVAR, "$.", :tCOMMA, ",", - :tGVAR, "$=", :tCOMMA, ",", - :tGVAR, "$:", :tCOMMA, ",", - :tGVAR, "$<", :tCOMMA, ",", - :tGVAR, "$>", :tCOMMA, ",", - :tGVAR, "$\"", - :tRBRACK, "]") + :tLBRACK, "[", [0, 1], + :tGVAR, "$~", [1, 3], :tCOMMA, ",", [3, 4], + :tGVAR, "$*", [5, 7], :tCOMMA, ",", [7, 8], + :tGVAR, "$$", [9, 11], :tCOMMA, ",", [11, 12], + :tGVAR, "$\?", [13, 15], :tCOMMA, ",", [15, 16], + :tGVAR, "$!", [17, 19], :tCOMMA, ",", [19, 20], + :tGVAR, "$@", [21, 23], :tCOMMA, ",", [23, 24], + :tGVAR, "$/", [25, 27], :tCOMMA, ",", [27, 28], + :tGVAR, "$\\", [29, 31], :tCOMMA, ",", [31, 32], + :tGVAR, "$;", [33, 35], :tCOMMA, ",", [35, 36], + :tGVAR, "$,", [37, 39], :tCOMMA, ",", [39, 40], + :tGVAR, "$.", [41, 43], :tCOMMA, ",", [43, 44], + :tGVAR, "$=", [45, 47], :tCOMMA, ",", [47, 48], + :tGVAR, "$:", [49, 51], :tCOMMA, ",", [51, 52], + :tGVAR, "$<", [53, 55], :tCOMMA, ",", [55, 56], + :tGVAR, "$>", [57, 59], :tCOMMA, ",", [59, 60], + :tGVAR, "$\"", [61, 63], + :tRBRACK, "]", [63, 64]) end def test_global_underscore assert_scanned("$_", - :tGVAR, "$_") + :tGVAR, "$_", [0, 2]) end - def test_global_wierd + def test_global_weird assert_scanned("$__blah", - :tGVAR, "$__blah") + :tGVAR, "$__blah", [0, 7]) end def test_global_zero - assert_scanned("$0", :tGVAR, "$0") + assert_scanned("$0", :tGVAR, "$0", [0, 2]) end def test_gt assert_scanned("a > 2", - :tIDENTIFIER, "a", - :tGT, ">", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tGT, ">", [2, 3], + :tINTEGER, 2, [4, 5]) end def test_heredoc_backtick assert_scanned("a = <<`EOF`\n blah blah\nEOF\n", - :tIDENTIFIER, "a", - :tEQL, "=", - :tXSTRING_BEG, "<<`", - :tSTRING_CONTENT, " blah blah\n", - :tSTRING_END, "EOF", - :tNL, nil) + :tIDENTIFIER, "a", [0, 1], + :tEQL, "=", [2, 3], + :tXSTRING_BEG, "<<`", [4, 11], + :tSTRING_CONTENT, " blah blah\n", [12, 24], + :tSTRING_END, "EOF", [24, 27], + :tNL, nil, [11, 12]) end def test_heredoc_double assert_scanned("a = <<\"EOF\"\n blah blah\nEOF\n", - :tIDENTIFIER, "a", - :tEQL, "=", - :tSTRING_BEG, "<<\"", - :tSTRING_CONTENT, " blah blah\n", - :tSTRING_END, "EOF", - :tNL, nil) + :tIDENTIFIER, "a", [0, 1], + :tEQL, "=", [2, 3], + :tSTRING_BEG, "<<\"", [4, 11], + :tSTRING_CONTENT, " blah blah\n", [12, 24], + :tSTRING_END, "EOF", [24, 27], + :tNL, nil, [11, 12]) end def test_heredoc_double_dash assert_scanned("a = <<-\"EOF\"\n blah blah\n EOF\n", - :tIDENTIFIER, "a", - :tEQL, "=", - :tSTRING_BEG, "<<\"", - :tSTRING_CONTENT, " blah blah\n", - :tSTRING_END, "EOF", - :tNL, nil) + :tIDENTIFIER, "a", [0, 1], + :tEQL, "=", [2, 3], + :tSTRING_BEG, "<<\"", [4, 12], + :tSTRING_CONTENT, " blah blah\n", [13, 25], + :tSTRING_END, "EOF", [25, 30], + :tNL, nil, [12, 13]) end def test_heredoc_double_eos refute_scanned("a = <<\"EOF\"\nblah", - :tIDENTIFIER, "a", - :tEQL, "=", - :tSTRING_BEG, "<<\"") + :tIDENTIFIER, "a", [0, 1], + :tEQL, "=", [2, 3], + :tSTRING_BEG, "<<\"", [4, 7]) end def test_heredoc_double_eos_nl refute_scanned("a = <<\"EOF\"\nblah\n", - :tIDENTIFIER, "a", - :tEQL, "=", - :tSTRING_BEG, "<<\"") + :tIDENTIFIER, "a", [0, 1], + :tEQL, "=", [2, 3], + :tSTRING_BEG, "<<\"", [4, 7]) end def test_heredoc_double_interp assert_scanned("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n", - :tIDENTIFIER, "a", - :tEQL, "=", - :tSTRING_BEG, "<<\"", - :tSTRING_CONTENT, "#x a ", - :tSTRING_DVAR, nil, - :tIVAR, "@a", - :tSTRING_CONTENT, " b ", - :tSTRING_DVAR, nil, - :tGVAR, "$b", - :tSTRING_CONTENT, " c ", - :tSTRING_DBEG, '#{', - :tINTEGER, 3, - :tRCURLY, "}", - :tSTRING_CONTENT, " \n", - :tSTRING_END, "EOF", - :tNL, nil) + :tIDENTIFIER, "a", [0, 1], + :tEQL, "=", [2, 3], + :tSTRING_BEG, "<<\"", [4, 11], + :tSTRING_CONTENT, "#x a ", [12, 17], + :tSTRING_DVAR, nil, [17, 18], + :tIVAR, "@a", [18, 20], + :tSTRING_CONTENT, " b ", [20, 23], + :tSTRING_DVAR, nil, [23, 24], + :tGVAR, "$b", [24, 26], + :tSTRING_CONTENT, " c ", [26, 29], + :tSTRING_DBEG, '#{', [29, 31], + :tINTEGER, 3, [31, 32], + :tRCURLY, "}", [32, 33], + :tSTRING_CONTENT, " \n", [33, 35], + :tSTRING_END, "EOF", [35, 38], + :tNL, nil, [11, 12]) end def test_heredoc_empty assert_scanned("<<\"\"\n\#{x}\nblah2\n\n", - :tSTRING_BEG, "<<\"", - :tSTRING_DBEG, "\#{", - :tIDENTIFIER, "x", - :tRCURLY, "}", - :tSTRING_CONTENT, "\n", - :tSTRING_CONTENT, "blah2\n", - :tSTRING_END, "", - :tNL, nil) + :tSTRING_BEG, "<<\"", [0, 4], + :tSTRING_DBEG, "\#{", [5, 7], + :tIDENTIFIER, "x", [7, 8], + :tRCURLY, "}", [8, 9], + :tSTRING_CONTENT, "\n", [9, 10], + :tSTRING_CONTENT, "blah2\n", [10, 16], + :tSTRING_END, "", [16, 16], + :tNL, nil, [4, 5]) end def test_heredoc_none assert_scanned("a = <", :tCMP + assert_lex_fname "<=>", :tCMP, [0, 3] end def test_identifier_def - assert_lex_fname "identifier", :tIDENTIFIER + assert_lex_fname "identifier", :tIDENTIFIER, [0, 10] end def test_identifier_eh - assert_scanned("identifier?", :tFID, "identifier?") + assert_scanned("identifier?", :tFID, "identifier?", [0, 11]) end def test_identifier_equals_arrow assert_scanned(":blah==>", - :tSYMBOL, "blah=", - :tASSOC, "=>") + :tSYMBOL, "blah=", [0, 6], + :tASSOC, "=>", [6, 8]) end def test_identifier_equals3 assert_scanned(":a===b", - :tSYMBOL, "a", - :tEQQ, "===", - :tIDENTIFIER, "b") + :tSYMBOL, "a", [0, 2], + :tEQQ, "===", [2, 5], + :tIDENTIFIER, "b", [5, 6]) end def test_identifier_equals_equals_arrow assert_scanned(":a==>b", - :tSYMBOL, "a=", - :tASSOC, "=>", - :tIDENTIFIER, "b") + :tSYMBOL, "a=", [0, 3], + :tASSOC, "=>", [3, 5], + :tIDENTIFIER, "b", [5, 6]) end def test_identifier_equals_caret - assert_lex_fname "^", :tCARET + assert_lex_fname "^", :tCARET, [0, 1] end def test_identifier_equals_def - assert_lex_fname "identifier=", :tIDENTIFIER + assert_lex_fname "identifier=", :tIDENTIFIER, [0, 11] end def test_identifier_equals_def2 - assert_lex_fname "==", :tEQ + assert_lex_fname "==", :tEQ, [0, 2] end def test_identifier_equals_expr @lex.state = :expr_dot assert_scanned("y = arg", - :tIDENTIFIER, "y", - :tEQL, "=", - :tIDENTIFIER, "arg") + :tIDENTIFIER, "y", [0, 1], + :tEQL, "=", [2, 3], + :tIDENTIFIER, "arg", [4, 7]) assert_equal :expr_arg, @lex.state end def test_identifier_equals_or - assert_lex_fname "|", :tPIPE + assert_lex_fname "|", :tPIPE, [0, 1] end def test_identifier_equals_slash - assert_lex_fname "/", :tDIVIDE + assert_lex_fname "/", :tDIVIDE, [0, 1] end def test_identifier_equals_tilde @lex.state = :expr_fname assert_scanned("identifier=~", - :tIDENTIFIER, "identifier=", - :tTILDE, "~") + :tIDENTIFIER, "identifier=", [0, 11], + :tTILDE, "~", [11, 12]) end def test_identifier_gt - assert_lex_fname ">", :tGT + assert_lex_fname ">", :tGT, [0, 1] end def test_identifier_le - assert_lex_fname "<=", :tLEQ + assert_lex_fname "<=", :tLEQ, [0, 2] end def test_identifier_lt - assert_lex_fname "<", :tLT + assert_lex_fname "<", :tLT, [0, 1] end def test_identifier_tilde - assert_lex_fname "~", :tTILDE + assert_lex_fname "~", :tTILDE, [0, 1] end def test_identifier_defined? - assert_lex_fname "defined?", :kDEFINED + assert_lex_fname "defined?", :kDEFINED, [0, 8] end def test_index - assert_lex_fname "[]", :tAREF + assert_lex_fname "[]", :tAREF, [0, 2] end def test_index_equals - assert_lex_fname "[]=", :tASET + assert_lex_fname "[]=", :tASET, [0, 3] end def test_integer - assert_scanned "42", :tINTEGER, 42 + assert_scanned "42", :tINTEGER, 42, [0, 2] end def test_integer_bin - assert_scanned "0b101010", :tINTEGER, 42 + assert_scanned "0b101010", :tINTEGER, 42, [0, 8] end def test_integer_bin_bad_none @@ -1073,7 +1078,7 @@ def test_integer_bin_bad_underscores end def test_integer_dec - assert_scanned "42", :tINTEGER, 42 + assert_scanned "42", :tINTEGER, 42, [0, 2] end def test_integer_dec_bad_underscores @@ -1081,7 +1086,7 @@ def test_integer_dec_bad_underscores end def test_integer_dec_d - assert_scanned "0d42", :tINTEGER, 42 + assert_scanned "0d42", :tINTEGER, 42, [0, 4] end def test_integer_dec_d_bad_none @@ -1095,25 +1100,25 @@ def test_integer_dec_d_bad_underscores def test_question_eh_a__18 setup_lexer 18 - assert_scanned "?a", :tINTEGER, 97 + assert_scanned "?a", :tINTEGER, 97, [0, 2] end def test_question_eh_a__19 setup_lexer 19 - assert_scanned '?a', :tCHARACTER, "a" + assert_scanned '?a', :tCHARACTER, "a", [0, 2] end def test_question_eh_escape_M_escape_C__18 setup_lexer 18 - assert_scanned '?\M-\C-a', :tINTEGER, 129 + assert_scanned '?\M-\C-a', :tINTEGER, 129, [0, 8] end def test_question_eh_escape_M_escape_C__19 setup_lexer 19 - assert_scanned '?\M-\C-a', :tCHARACTER, "\M-\C-a" + assert_scanned '?\M-\C-a', :tCHARACTER, "\M-\C-a", [0, 8] end def test_question_eh_escape_u_1_digit @@ -1137,17 +1142,17 @@ def test_question_eh_escape_u_3_digits def test_question_eh_escape_u_4_digits if RUBY_VERSION >= '1.9' setup_lexer 19 - assert_scanned '?\\u0001', :tCHARACTER, "\u0001" + assert_scanned '?\\u0001', :tCHARACTER, "\u0001", [0, 7] end end def test_question_eh_single_unicode_point if RUBY_VERSION >= '1.9' setup_lexer 19 - assert_scanned '?\\u{123}', :tCHARACTER, "\u0123" + assert_scanned '?\\u{123}', :tCHARACTER, "\u0123", [0, 8] setup_lexer 19 - assert_scanned '?\\u{a}', :tCHARACTER, "\n" + assert_scanned '?\\u{a}', :tCHARACTER, "\n", [0, 6] end end @@ -1166,7 +1171,7 @@ def test_question_eh_escape_u_unclosed_bracket end def test_integer_hex - assert_scanned "0x2a", :tINTEGER, 42 + assert_scanned "0x2a", :tINTEGER, 42, [0, 4] end def test_integer_hex_bad_none @@ -1178,7 +1183,7 @@ def test_integer_hex_bad_underscores end def test_integer_oct - assert_scanned "052", :tINTEGER, 42 + assert_scanned "052", :tINTEGER, 42, [0, 3] end def test_integer_oct_bad_range @@ -1190,7 +1195,7 @@ def test_integer_oct_bad_underscores end def test_integer_oct_O - assert_scanned "0O52", :tINTEGER, 42 + assert_scanned "0O52", :tINTEGER, 42, [0, 4] end def test_integer_oct_O_bad_range @@ -1202,11 +1207,11 @@ def test_integer_oct_O_bad_underscores end def test_integer_oct_O_not_bad_none - assert_scanned "0O ", :tINTEGER, 0 + assert_scanned "0O ", :tINTEGER, 0, [0, 2] end def test_integer_oct_o - assert_scanned "0o52", :tINTEGER, 42 + assert_scanned "0o52", :tINTEGER, 42, [0, 4] end def test_integer_oct_o_bad_range @@ -1218,18 +1223,18 @@ def test_integer_oct_o_bad_underscores end def test_integer_oct_o_not_bad_none - assert_scanned "0o ", :tINTEGER, 0 + assert_scanned "0o ", :tINTEGER, 0, [0, 2] end def test_integer_trailing assert_scanned("1.to_s", - :tINTEGER, 1, - :tDOT, '.', - :tIDENTIFIER, 'to_s') + :tINTEGER, 1, [0, 1], + :tDOT, '.', [1, 2], + :tIDENTIFIER, 'to_s', [2, 6]) end def test_integer_underscore - assert_scanned "4_2", :tINTEGER, 42 + assert_scanned "4_2", :tINTEGER, 42, [0, 3] end def test_integer_underscore_bad @@ -1237,11 +1242,11 @@ def test_integer_underscore_bad end def test_integer_zero - assert_scanned "0", :tINTEGER, 0 + assert_scanned "0", :tINTEGER, 0, [0, 1] end def test_ivar - assert_scanned "@blah", :tIVAR, "@blah" + assert_scanned "@blah", :tIVAR, "@blah", [0, 5] end def test_ivar_bad @@ -1249,215 +1254,215 @@ def test_ivar_bad end def test_ivar_bad_0_length - refute_scanned "1+@\n", :tINTEGER, 1, :tPLUS, "+" + refute_scanned "1+@\n", :tINTEGER, 1, [0, 1], :tPLUS, "+", [1, 2] end def test_keyword_expr @lex.state = :expr_endarg - assert_scanned("if", :kIF_MOD, "if") + assert_scanned "if", :kIF_MOD, "if", [0, 2] assert_equal :expr_beg, @lex.state end def test_lt - assert_scanned "<", :tLT, "<" + assert_scanned "<", :tLT, "<", [0, 1] end def test_lt2 assert_scanned("a <\< b", - :tIDENTIFIER, "a", - :tLSHFT, "<\<", - :tIDENTIFIER, "b") + :tIDENTIFIER, "a", [0, 1], + :tLSHFT, "<\<", [2, 4], + :tIDENTIFIER, "b", [5, 6]) end def test_lt2_equals assert_scanned("a <\<= b", - :tIDENTIFIER, "a", - :tOP_ASGN, "<\<", - :tIDENTIFIER, "b") + :tIDENTIFIER, "a", [0, 1], + :tOP_ASGN, "<\<", [2, 5], + :tIDENTIFIER, "b", [6, 7]) end def test_lt_equals - assert_scanned "<=", :tLEQ, "<=" + assert_scanned "<=", :tLEQ, "<=", [0, 2] end def test_minus assert_scanned("1 - 2", - :tINTEGER, 1, - :tMINUS, "-", - :tINTEGER, 2) + :tINTEGER, 1, [0, 1], + :tMINUS, "-", [2, 3], + :tINTEGER, 2, [4, 5]) end def test_minus_equals @lex.state = :expr_end - assert_scanned "-=", :tOP_ASGN, "-" + assert_scanned "-=", :tOP_ASGN, "-", [0, 2] end def test_minus_method @lex.state = :expr_fname - assert_scanned "-", :tMINUS, "-" + assert_scanned "-", :tMINUS, "-", [0, 1] end def test_minus_unary_method @lex.state = :expr_fname - assert_scanned "-@", :tUMINUS, "-@" + assert_scanned "-@", :tUMINUS, "-@", [0, 2] end def test_minus_unary_number assert_scanned("-42", - :tUMINUS_NUM, "-", - :tINTEGER, 42) + :tUMINUS_NUM, "-", [0, 1], + :tINTEGER, 42, [1, 3]) end def test_nth_ref assert_scanned('[$1, $2, $3]', - :tLBRACK, "[", - :tNTH_REF, 1, :tCOMMA, ",", - :tNTH_REF, 2, :tCOMMA, ",", - :tNTH_REF, 3, - :tRBRACK, "]") + :tLBRACK, "[", [0, 1], + :tNTH_REF, 1, [1, 3], :tCOMMA, ",", [3, 4], + :tNTH_REF, 2, [5, 7], :tCOMMA, ",", [7, 8], + :tNTH_REF, 3, [9, 11], + :tRBRACK, "]", [11, 12]) end def test_open_bracket - assert_scanned("(", :tLPAREN, "(") + assert_scanned("(", :tLPAREN, "(", [0, 1]) end def test_open_bracket_cmdarg - assert_scanned("m (", :tIDENTIFIER, "m", - :tLPAREN_ARG, "(") + assert_scanned("m (", :tIDENTIFIER, "m", [0, 1], + :tLPAREN_ARG, "(", [2, 3]) end def test_open_bracket_exprarg - assert_scanned("m(", :tIDENTIFIER, "m", - :tLPAREN2, "(") + assert_scanned("m(", :tIDENTIFIER, "m", [0, 1], + :tLPAREN2, "(", [1, 2]) end def test_open_curly_bracket assert_scanned("{", - :tLBRACE, "{") + :tLBRACE, "{", [0, 1]) end def test_open_curly_bracket_arg assert_scanned("m { 3 }", - :tIDENTIFIER, "m", - :tLCURLY, "{", - :tINTEGER, 3, - :tRCURLY, "}") + :tIDENTIFIER, "m", [0, 1], + :tLCURLY, "{", [2, 3], + :tINTEGER, 3, [4, 5], + :tRCURLY, "}", [6, 7]) end def test_open_curly_bracket_block @lex.state = :expr_endarg # seen m(3) assert_scanned("{ 4 }", - :tLBRACE_ARG, "{", - :tINTEGER, 4, - :tRCURLY, "}") + :tLBRACE_ARG, "{", [0, 1], + :tINTEGER, 4, [2, 3], + :tRCURLY, "}", [4, 5]) end def test_open_square_bracket_arg assert_scanned("m [ 3 ]", - :tIDENTIFIER, "m", - :tLBRACK, "[", - :tINTEGER, 3, - :tRBRACK, "]") + :tIDENTIFIER, "m", [0, 1], + :tLBRACK, "[", [2, 3], + :tINTEGER, 3, [4, 5], + :tRBRACK, "]", [6, 7]) end def test_open_square_bracket_ary assert_scanned("[1, 2, 3]", - :tLBRACK, "[", - :tINTEGER, 1, - :tCOMMA, ",", - :tINTEGER, 2, - :tCOMMA, ",", - :tINTEGER, 3, - :tRBRACK, "]") + :tLBRACK, "[", [0, 1], + :tINTEGER, 1, [1, 2], + :tCOMMA, ",", [2, 3], + :tINTEGER, 2, [4, 5], + :tCOMMA, ",", [5, 6], + :tINTEGER, 3, [7, 8], + :tRBRACK, "]", [8, 9]) end def test_open_square_bracket_meth assert_scanned("m[3]", - :tIDENTIFIER, "m", - :tLBRACK2, "[", - :tINTEGER, 3, - :tRBRACK, "]") + :tIDENTIFIER, "m", [0, 1], + :tLBRACK2, "[", [1, 2], + :tINTEGER, 3, [2, 3], + :tRBRACK, "]", [3, 4]) end def test_or - assert_scanned "|", :tPIPE, "|" + assert_scanned "|", :tPIPE, "|", [0, 1] end def test_or2 - assert_scanned "||", :tOROP, "||" + assert_scanned "||", :tOROP, "||", [0, 2] end def test_or2_equals - assert_scanned "||=", :tOP_ASGN, "||" + assert_scanned "||=", :tOP_ASGN, "||", [0, 3] end def test_or_equals - assert_scanned "|=", :tOP_ASGN, "|" + assert_scanned "|=", :tOP_ASGN, "|", [0, 2] end def test_percent assert_scanned("a % 2", - :tIDENTIFIER, "a", - :tPERCENT, "%", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tPERCENT, "%", [2, 3], + :tINTEGER, 2, [4, 5]) end def test_percent_equals assert_scanned("a %= 2", - :tIDENTIFIER, "a", - :tOP_ASGN, "%", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tOP_ASGN, "%", [2, 4], + :tINTEGER, 2, [5, 6]) end def test_plus assert_scanned("1 + 1", - :tINTEGER, 1, - :tPLUS, "+", - :tINTEGER, 1) + :tINTEGER, 1, [0, 1], + :tPLUS, "+", [2, 3], + :tINTEGER, 1, [4, 5]) end def test_plus_equals @lex.state = :expr_end - assert_scanned "+=", :tOP_ASGN, "+" + assert_scanned "+=", :tOP_ASGN, "+", [0, 2] end def test_plus_method @lex.state = :expr_fname - assert_scanned "+", :tPLUS, "+" + assert_scanned "+", :tPLUS, "+", [0, 1] end def test_plus_unary_method @lex.state = :expr_fname - assert_scanned "+@", :tUPLUS, "+@" + assert_scanned "+@", :tUPLUS, "+@", [0, 2] end def test_numbers - assert_scanned "0b10", :tINTEGER, 2 - assert_scanned "0B10", :tINTEGER, 2 + assert_scanned "0b10", :tINTEGER, 2, [0, 4] + assert_scanned "0B10", :tINTEGER, 2, [0, 4] - assert_scanned "0d10", :tINTEGER, 10 - assert_scanned "0D10", :tINTEGER, 10 + assert_scanned "0d10", :tINTEGER, 10, [0, 4] + assert_scanned "0D10", :tINTEGER, 10, [0, 4] - assert_scanned "0x10", :tINTEGER, 16 - assert_scanned "0X10", :tINTEGER, 16 + assert_scanned "0x10", :tINTEGER, 16, [0, 4] + assert_scanned "0X10", :tINTEGER, 16, [0, 4] - assert_scanned "0o10", :tINTEGER, 8 - assert_scanned "0O10", :tINTEGER, 8 - assert_scanned "0o", :tINTEGER, 0 - assert_scanned "0O", :tINTEGER, 0 + assert_scanned "0o10", :tINTEGER, 8, [0, 4] + assert_scanned "0O10", :tINTEGER, 8, [0, 4] + assert_scanned "0o", :tINTEGER, 0, [0, 2] + assert_scanned "0O", :tINTEGER, 0, [0, 2] - assert_scanned "0o", :tINTEGER, 0 - assert_scanned "0O", :tINTEGER, 0 + assert_scanned "0o", :tINTEGER, 0, [0, 2] + assert_scanned "0O", :tINTEGER, 0, [0, 2] - assert_scanned "0777_333", :tINTEGER, 261851 + assert_scanned "0777_333", :tINTEGER, 261851, [0, 8] - assert_scanned "0", :tINTEGER, 0 + assert_scanned "0", :tINTEGER, 0, [0, 1] refute_scanned "0x" refute_scanned "0X" @@ -1483,19 +1488,19 @@ def test_numbers def test_plus_unary_number assert_scanned("+42", - :tINTEGER, 42) + :tINTEGER, 42, [1, 3]) end def test_question__18 setup_lexer 18 - assert_scanned "?*", :tINTEGER, 42 + assert_scanned "?*", :tINTEGER, 42, [0, 2] end def test_question__19 setup_lexer 19 - assert_scanned "?*", :tCHARACTER, "*" + assert_scanned "?*", :tCHARACTER, "*", [0, 2] end def test_question_bad_eos @@ -1503,369 +1508,369 @@ def test_question_bad_eos end def test_question_bad_ws - assert_scanned "? ", :tEH, "?" - assert_scanned "?\n", :tEH, "?" - assert_scanned "?\t", :tEH, "?" - assert_scanned "?\v", :tEH, "?" - assert_scanned "?\r", :tEH, "?" - assert_scanned "?\f", :tEH, "?" + assert_scanned "? ", :tEH, "?", [0, 1] + assert_scanned "?\n", :tEH, "?", [0, 1] + assert_scanned "?\t", :tEH, "?", [0, 1] + assert_scanned "?\v", :tEH, "?", [0, 1] + assert_scanned "?\r", :tEH, "?", [0, 1] + assert_scanned "?\f", :tEH, "?", [0, 1] end def test_question_ws_backslashed__18 setup_lexer 18 @lex.state = :expr_beg - assert_scanned "?\\ ", :tINTEGER, 32 + assert_scanned "?\\ ", :tINTEGER, 32, [0, 3] @lex.state = :expr_beg - assert_scanned "?\\n", :tINTEGER, 10 + assert_scanned "?\\n", :tINTEGER, 10, [0, 3] @lex.state = :expr_beg - assert_scanned "?\\t", :tINTEGER, 9 + assert_scanned "?\\t", :tINTEGER, 9, [0, 3] @lex.state = :expr_beg - assert_scanned "?\\v", :tINTEGER, 11 + assert_scanned "?\\v", :tINTEGER, 11, [0, 3] @lex.state = :expr_beg - assert_scanned "?\\r", :tINTEGER, 13 + assert_scanned "?\\r", :tINTEGER, 13, [0, 3] @lex.state = :expr_beg - assert_scanned "?\\f", :tINTEGER, 12 + assert_scanned "?\\f", :tINTEGER, 12, [0, 3] end def test_question_ws_backslashed__19 setup_lexer 19 @lex.state = :expr_beg - assert_scanned "?\\ ", :tCHARACTER, " " + assert_scanned "?\\ ", :tCHARACTER, " ", [0, 3] @lex.state = :expr_beg - assert_scanned "?\\n", :tCHARACTER, "\n" + assert_scanned "?\\n", :tCHARACTER, "\n", [0, 3] @lex.state = :expr_beg - assert_scanned "?\\t", :tCHARACTER, "\t" + assert_scanned "?\\t", :tCHARACTER, "\t", [0, 3] @lex.state = :expr_beg - assert_scanned "?\\v", :tCHARACTER, "\v" + assert_scanned "?\\v", :tCHARACTER, "\v", [0, 3] @lex.state = :expr_beg - assert_scanned "?\\r", :tCHARACTER, "\r" + assert_scanned "?\\r", :tCHARACTER, "\r", [0, 3] @lex.state = :expr_beg - assert_scanned "?\\f", :tCHARACTER, "\f" + assert_scanned "?\\f", :tCHARACTER, "\f", [0, 3] end def test_rbracket - assert_scanned "]", :tRBRACK, "]" + assert_scanned "]", :tRBRACK, "]", [0, 1] end def test_rcurly - assert_scanned "}", :tRCURLY, "}" + assert_scanned "}", :tRCURLY, "}", [0, 1] end def test_regexp assert_scanned("/regexp/", - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regexp", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regexp", [1, 7], + :tSTRING_END, "/", [7, 8], + :tREGEXP_OPT, "", [8, 8]) end def test_regexp_ambiguous assert_scanned("method /regexp/", - :tIDENTIFIER, "method", - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regexp", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tIDENTIFIER, "method", [0, 6], + :tREGEXP_BEG, "/", [7, 8], + :tSTRING_CONTENT, "regexp", [8, 14], + :tSTRING_END, "/", [14, 15], + :tREGEXP_OPT, "", [15, 15]) end def test_regexp_bad refute_scanned("/.*/xyz", - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, ".*", - :tSTRING_END, "/") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, ".*", [1, 3], + :tSTRING_END, "/", [3, 4]) end def test_regexp_escape_C assert_scanned('/regex\\C-x/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\C-x", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\C-x", [1, 10], + :tSTRING_END, "/", [10, 11], + :tREGEXP_OPT, "", [11, 11]) end def test_regexp_escape_C_M assert_scanned('/regex\\C-\\M-x/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\C-\\M-x", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\C-\\M-x", [1, 13], + :tSTRING_END, "/", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_C_M_craaaazy assert_scanned("/regex\\C-\\\n\\M-x/", - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\C-\\M-x", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\C-\\M-x", [1, 15], + :tSTRING_END, "/", [15, 16], + :tREGEXP_OPT, "", [16, 16]) end def test_regexp_escape_C_bad_dash - refute_scanned '/regex\\Cx/', :tREGEXP_BEG, "/" + refute_scanned '/regex\\Cx/', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_C_bad_dash_eos - refute_scanned '/regex\\C-/', :tREGEXP_BEG, "/" + refute_scanned '/regex\\C-/', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_C_bad_dash_eos2 - refute_scanned '/regex\\C-', :tREGEXP_BEG, "/" + refute_scanned '/regex\\C-', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_C_bad_eos - refute_scanned '/regex\\C/', :tREGEXP_BEG, "/" + refute_scanned '/regex\\C/', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_C_bad_eos2 - refute_scanned '/regex\\c', :tREGEXP_BEG, "/" + refute_scanned '/regex\\c', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_M assert_scanned('/regex\\M-x/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\M-x", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\M-x", [1, 10], + :tSTRING_END, "/", [10, 11], + :tREGEXP_OPT, "", [11, 11]) end def test_regexp_escape_M_C assert_scanned('/regex\\M-\\C-x/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\M-\\C-x", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\M-\\C-x", [1, 13], + :tSTRING_END, "/", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_M_bad_dash - refute_scanned '/regex\\Mx/', :tREGEXP_BEG, "/" + refute_scanned '/regex\\Mx/', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_M_bad_dash_eos - refute_scanned '/regex\\M-/', :tREGEXP_BEG, "/" + refute_scanned '/regex\\M-/', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_M_bad_dash_eos2 - refute_scanned '/regex\\M-', :tREGEXP_BEG, "/" + refute_scanned '/regex\\M-', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_M_bad_eos - refute_scanned '/regex\\M/', :tREGEXP_BEG, "/" + refute_scanned '/regex\\M/', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_backslash_slash assert_scanned('/\\//', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, '/', - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, '/', [1, 3], + :tSTRING_END, "/", [3, 4], + :tREGEXP_OPT, "", [4, 4]) end def test_regexp_escape_backslash_terminator assert_scanned('%r%blah\\%blah%', - :tREGEXP_BEG, "%r%", - :tSTRING_CONTENT, "blah%blah", - :tSTRING_END, "%", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "%r%", [0, 3], + :tSTRING_CONTENT, "blah%blah", [3, 13], + :tSTRING_END, "%", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_backslash_terminator_meta1 assert_scanned('%r{blah\\}blah}', - :tREGEXP_BEG, "%r{", - :tSTRING_CONTENT, "blah\\}blah", - :tSTRING_END, "}", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "%r{", [0, 3], + :tSTRING_CONTENT, "blah\\}blah", [3, 13], + :tSTRING_END, "}", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_backslash_terminator_meta2 assert_scanned('%r/blah\\/blah/', - :tREGEXP_BEG, "%r/", - :tSTRING_CONTENT, "blah/blah", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "%r/", [0, 3], + :tSTRING_CONTENT, "blah/blah", [3, 13], + :tSTRING_END, "/", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_backslash_terminator_meta3 assert_scanned('%r/blah\\%blah/', - :tREGEXP_BEG, "%r/", - :tSTRING_CONTENT, "blah\\%blah", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "%r/", [0, 3], + :tSTRING_CONTENT, "blah\\%blah", [3, 13], + :tSTRING_END, "/", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_bad_eos - refute_scanned '/regex\\', :tREGEXP_BEG, "/" + refute_scanned '/regex\\', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_bs assert_scanned('/regex\\\\regex/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\\\regex", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\\\regex", [1, 13], + :tSTRING_END, "/", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_c assert_scanned('/regex\\cxxx/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\cxxx", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\cxxx", [1, 11], + :tSTRING_END, "/", [11, 12], + :tREGEXP_OPT, "", [12, 12]) end def test_regexp_escape_c_backslash assert_scanned('/regex\\c\\n/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\c\\n", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\c\\n", [1, 10], + :tSTRING_END, "/", [10, 11], + :tREGEXP_OPT, "", [11, 11]) end def test_regexp_escape_chars assert_scanned('/re\\tge\\nxp/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "re\\tge\\nxp", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "re\\tge\\nxp", [1, 11], + :tSTRING_END, "/", [11, 12], + :tREGEXP_OPT, "", [12, 12]) end def test_regexp_escape_double_backslash assert_scanned('/[\\/\\\\]$/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT,'[/\\\\]$', - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT,'[/\\\\]$', [1, 8], + :tSTRING_END, "/", [8, 9], + :tREGEXP_OPT, "", [9, 9]) end def test_regexp_escape_hex assert_scanned('/regex\\x61xp/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\x61xp", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\x61xp", [1, 12], + :tSTRING_END, "/", [12, 13], + :tREGEXP_OPT, "", [13, 13]) end def test_regexp_escape_hex_bad - refute_scanned '/regex\\xzxp/', :tREGEXP_BEG, "/" + refute_scanned '/regex\\xzxp/', :tREGEXP_BEG, "/", [0, 1] end def test_regexp_escape_hex_one assert_scanned('/^[\\xd\\xa]{2}/on', - :tREGEXP_BEG, '/', - :tSTRING_CONTENT, '^[\\xd\\xa]{2}', - :tSTRING_END, "/", - :tREGEXP_OPT, 'on') + :tREGEXP_BEG, '/', [0, 1], + :tSTRING_CONTENT, '^[\\xd\\xa]{2}', [1, 13], + :tSTRING_END, "/", [13, 14], + :tREGEXP_OPT, 'on', [14, 16]) end def test_regexp_escape_oct1 assert_scanned('/regex\\0xp/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\0xp", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\0xp", [1, 10], + :tSTRING_END, "/", [10, 11], + :tREGEXP_OPT, "", [11, 11]) end def test_regexp_escape_oct2 assert_scanned('/regex\\07xp/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\07xp", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\07xp", [1, 11], + :tSTRING_END, "/", [11, 12], + :tREGEXP_OPT, "", [12, 12]) end def test_regexp_escape_oct3 assert_scanned('/regex\\10142/', - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regex\\10142", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regex\\10142", [1, 12], + :tSTRING_END, "/", [12, 13], + :tREGEXP_OPT, "", [13, 13]) end def test_regexp_escape_return assert_scanned("/regex\\\nregex/", - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "regexregex", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "regexregex", [1, 13], + :tSTRING_END, "/", [13, 14], + :tREGEXP_OPT, "", [14, 14]) end def test_regexp_escape_delimiter_meta assert_scanned("%r(\\))", - :tREGEXP_BEG, "%r(", - :tSTRING_CONTENT, "\\)", - :tSTRING_END, ")", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "%r(", [0, 3], + :tSTRING_CONTENT, "\\)", [3, 5], + :tSTRING_END, ")", [5, 6], + :tREGEXP_OPT, "", [6, 6]) end def test_regexp_escape_delimiter_nonmeta assert_scanned("%r'\\''", - :tREGEXP_BEG, "%r'", - :tSTRING_CONTENT, "'", - :tSTRING_END, "'", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "%r'", [0, 3], + :tSTRING_CONTENT, "'", [3, 5], + :tSTRING_END, "'", [5, 6], + :tREGEXP_OPT, "", [6, 6]) end def test_regexp_escape_other_meta assert_scanned("/\\.\\$\\*\\+\\.\\?\\|/", - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, "\\.\\$\\*\\+\\.\\?\\|", - :tSTRING_END, "/", - :tREGEXP_OPT, "") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, "\\.\\$\\*\\+\\.\\?\\|", [1, 15], + :tSTRING_END, "/", [15, 16], + :tREGEXP_OPT, "", [16, 16]) end def test_regexp_nm assert_scanned("/.*/nm", - :tREGEXP_BEG, "/", - :tSTRING_CONTENT, ".*", - :tSTRING_END, "/", - :tREGEXP_OPT, "nm") + :tREGEXP_BEG, "/", [0, 1], + :tSTRING_CONTENT, ".*", [1, 3], + :tSTRING_END, "/", [3, 4], + :tREGEXP_OPT, "nm", [4, 6]) end def test_rparen - assert_scanned ")", :tRPAREN, ")" + assert_scanned ")", :tRPAREN, ")", [0, 1] end def test_rshft assert_scanned("a >> 2", - :tIDENTIFIER, "a", - :tRSHFT, ">>", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tRSHFT, ">>", [2, 4], + :tINTEGER, 2, [5, 6]) end def test_rshft_equals assert_scanned("a >>= 2", - :tIDENTIFIER, "a", - :tOP_ASGN, ">>", - :tINTEGER, 2) + :tIDENTIFIER, "a", [0, 1], + :tOP_ASGN, ">>", [2, 5], + :tINTEGER, 2, [6, 7]) end def test_star assert_scanned("a * ", - :tIDENTIFIER, "a", - :tSTAR2, "*") + :tIDENTIFIER, "a", [0, 1], + :tSTAR2, "*", [2, 3]) assert_equal :expr_beg, @lex.state end def test_star2 assert_scanned("a ** ", - :tIDENTIFIER, "a", - :tPOW, "**") + :tIDENTIFIER, "a", [0, 1], + :tPOW, "**", [2, 4]) assert_equal :expr_beg, @lex.state end def test_star2_equals assert_scanned("a **= ", - :tIDENTIFIER, "a", - :tOP_ASGN, "**") + :tIDENTIFIER, "a", [0, 1], + :tOP_ASGN, "**", [2, 5]) assert_equal :expr_beg, @lex.state end def test_star2_beg assert_scanned("** ", - :tDSTAR, "**") + :tDSTAR, "**", [0, 2]) assert_equal :expr_beg, @lex.state end @@ -1874,8 +1879,8 @@ def test_star_arg @lex.state = :expr_arg assert_scanned(" *a", - :tSTAR, "*", - :tIDENTIFIER, "a") + :tSTAR, "*", [1, 2], + :tIDENTIFIER, "a", [2, 3]) assert_equal :expr_arg, @lex.state end @@ -1884,8 +1889,8 @@ def test_star_arg_beg @lex.state = :expr_beg assert_scanned("*a", - :tSTAR, "*", - :tIDENTIFIER, "a") + :tSTAR, "*", [0, 1], + :tIDENTIFIER, "a", [1, 2]) assert_equal :expr_arg, @lex.state end @@ -1894,454 +1899,454 @@ def test_star_arg_beg_fname @lex.state = :expr_fname assert_scanned("*a", - :tSTAR2, "*", - :tIDENTIFIER, "a") + :tSTAR2, "*", [0, 1], + :tIDENTIFIER, "a", [1, 2]) assert_equal :expr_arg, @lex.state end def test_star_equals assert_scanned("a *= ", - :tIDENTIFIER, "a", - :tOP_ASGN, "*") + :tIDENTIFIER, "a", [0, 1], + :tOP_ASGN, "*", [2, 4]) assert_equal :expr_beg, @lex.state end def test_string_bad_eos refute_scanned('%', - :tSTRING_BEG, '%') + :tSTRING_BEG, '%', [0, 1]) end def test_string_bad_eos_quote refute_scanned('%{nest', - :tSTRING_BEG, '%}') + :tSTRING_BEG, '%}', [0, 2]) end def test_string_double assert_scanned('"string"', - :tSTRING, "string") + :tSTRING, "string", [0, 8]) end def test_string_double_escape_C assert_scanned('"\\C-a"', - :tSTRING, "\001") + :tSTRING, "\001", [0, 6]) end def test_string_double_escape_C_backslash assert_scanned('"\\C-\\\\"', - :tSTRING, "\034") + :tSTRING, "\034", [0, 7]) end def test_string_double_escape_C_escape assert_scanned('"\\C-\\M-a"', - :tSTRING, "\201") + :tSTRING, "\201", [0, 9]) end def test_string_double_escape_C_question assert_scanned('"\\C-?"', - :tSTRING, "\177") + :tSTRING, "\177", [0, 6]) end def test_string_double_escape_M assert_scanned('"\\M-a"', - :tSTRING, "\341") + :tSTRING, "\341", [0, 6]) end def test_string_double_escape_M_backslash assert_scanned('"\\M-\\\\"', - :tSTRING, "\334") + :tSTRING, "\334", [0, 7]) end def test_string_double_escape_M_escape assert_scanned('"\\M-\\C-a"', - :tSTRING, "\201") + :tSTRING, "\201", [0, 9]) end def test_string_double_escape_bs1 assert_scanned('"a\\a\\a"', - :tSTRING, "a\a\a") + :tSTRING, "a\a\a", [0, 7]) end def test_string_double_escape_bs2 assert_scanned('"a\\\\a"', - :tSTRING, "a\\a") + :tSTRING, "a\\a", [0, 6]) end def test_string_double_escape_c assert_scanned('"\\ca"', - :tSTRING, "\001") + :tSTRING, "\001", [0, 5]) end def test_string_double_escape_c_escape assert_scanned('"\\c\\M-a"', - :tSTRING, "\201") + :tSTRING, "\201", [0, 8]) end def test_string_double_escape_c_question assert_scanned('"\\c?"', - :tSTRING, "\177") + :tSTRING, "\177", [0, 5]) end def test_string_double_escape_chars assert_scanned('"s\\tri\\ng"', - :tSTRING, "s\tri\ng") + :tSTRING, "s\tri\ng", [0, 10]) end def test_string_double_escape_hex assert_scanned('"n = \\x61\\x62\\x63"', - :tSTRING, "n = abc") + :tSTRING, "n = abc", [0, 18]) end def test_string_double_escape_octal assert_scanned('"n = \\101\\102\\103"', - :tSTRING, "n = ABC") + :tSTRING, "n = ABC", [0, 18]) end def test_string_double_escape_octal_wrap assert_scanned('"\\753"', - :tSTRING, "\xEB") + :tSTRING, "\xEB", [0, 6]) end def test_string_double_interp assert_scanned("\"blah #x a \#@a b \#$b c \#{3} # \"", - :tSTRING_BEG, "\"", - :tSTRING_CONTENT, "blah #x a ", - :tSTRING_DVAR, nil, - :tIVAR, "@a", - :tSTRING_CONTENT, " b ", - :tSTRING_DVAR, nil, - :tGVAR, "$b", - :tSTRING_CONTENT, " c ", - :tSTRING_DBEG, '#{', - :tINTEGER, 3, - :tRCURLY, "}", - :tSTRING_CONTENT, " # ", - :tSTRING_END, "\"") + :tSTRING_BEG, "\"", [0, 1], + :tSTRING_CONTENT, "blah #x a ", [1, 11], + :tSTRING_DVAR, nil, [11, 12], + :tIVAR, "@a", [12, 14], + :tSTRING_CONTENT, " b ", [14, 17], + :tSTRING_DVAR, nil, [17, 18], + :tGVAR, "$b", [18, 20], + :tSTRING_CONTENT, " c ", [20, 23], + :tSTRING_DBEG, '#{', [23, 25], + :tINTEGER, 3, [25, 26], + :tRCURLY, "}", [26, 27], + :tSTRING_CONTENT, " # ", [27, 30], + :tSTRING_END, "\"", [30, 31]) end def test_string_double_interp_label assert_scanned('"#{foo:bar}"', - :tSTRING_BEG, '"', - :tSTRING_DBEG, '#{', - :tIDENTIFIER, 'foo', - :tSYMBOL, 'bar', - :tRCURLY, '}', - :tSTRING_END, '"') + :tSTRING_BEG, '"', [0, 1], + :tSTRING_DBEG, '#{', [1, 3], + :tIDENTIFIER, 'foo', [3, 6], + :tSYMBOL, 'bar', [6, 10], + :tRCURLY, '}', [10, 11], + :tSTRING_END, '"', [11, 12]) end def test_string_double_nested_curlies assert_scanned('%{nest{one{two}one}nest}', - :tSTRING_BEG, '%{', - :tSTRING_CONTENT, "nest{one{two}one}nest", - :tSTRING_END, '}') + :tSTRING_BEG, '%{', [0, 2], + :tSTRING_CONTENT, "nest{one{two}one}nest", [2, 23], + :tSTRING_END, '}', [23, 24]) end def test_string_double_no_interp assert_scanned("\"# blah\"", # pound first - :tSTRING, "# blah") + :tSTRING, "# blah", [0, 8]) assert_scanned("\"blah # blah\"", # pound not first - :tSTRING, "blah # blah") + :tSTRING, "blah # blah", [0, 13]) end def test_string_escape_x_single assert_scanned('"\\x0"', - :tSTRING, "\000") + :tSTRING, "\000", [0, 5]) end def test_string_pct_Q assert_scanned("%Q[s1 s2]", - :tSTRING_BEG, '%Q[', - :tSTRING_CONTENT, "s1 s2", - :tSTRING_END, ']') + :tSTRING_BEG, '%Q[', [0, 3], + :tSTRING_CONTENT, "s1 s2", [3, 8], + :tSTRING_END, ']', [8, 9]) end def test_string_pct_W assert_scanned("%W[s1 s2\ns3]", - :tWORDS_BEG, "%W[", - :tSTRING_CONTENT, "s1", - :tSPACE, nil, - :tSTRING_CONTENT, "s2", - :tSPACE, nil, - :tSTRING_CONTENT, "s3", - :tSPACE, nil, - :tSTRING_END, ']') + :tWORDS_BEG, "%W[", [0, 3], + :tSTRING_CONTENT, "s1", [3, 5], + :tSPACE, nil, [5, 6], + :tSTRING_CONTENT, "s2", [6, 8], + :tSPACE, nil, [8, 9], + :tSTRING_CONTENT, "s3", [9, 11], + :tSPACE, nil, [11, 11], + :tSTRING_END, ']', [11, 12]) end def test_string_pct_W_bs_nl assert_scanned("%W[s1 \\\ns2]", - :tWORDS_BEG, "%W[", - :tSTRING_CONTENT, "s1", - :tSPACE, nil, - :tSTRING_CONTENT, "\ns2", - :tSPACE, nil, - :tSTRING_END, ']') + :tWORDS_BEG, "%W[", [0, 3], + :tSTRING_CONTENT, "s1", [3, 5], + :tSPACE, nil, [5, 6], + :tSTRING_CONTENT, "\ns2", [6, 10], + :tSPACE, nil, [10, 10], + :tSTRING_END, ']', [10, 11]) end def test_string_pct_W_interp assert_scanned('%W[#{1}#{2} #@a]', - :tWORDS_BEG, '%W[', - :tSTRING_DBEG, '#{', - :tINTEGER, 1, - :tRCURLY, '}', - :tSTRING_DBEG, '#{', - :tINTEGER, 2, - :tRCURLY, '}', - :tSPACE, nil, - :tSTRING_DVAR, nil, - :tIVAR, '@a', - :tSPACE, nil, - :tSTRING_END, ']') + :tWORDS_BEG, '%W[', [0, 3], + :tSTRING_DBEG, '#{', [3, 5], + :tINTEGER, 1, [5, 6], + :tRCURLY, '}', [6, 7], + :tSTRING_DBEG, '#{', [7, 9], + :tINTEGER, 2, [9, 10], + :tRCURLY, '}', [10, 11], + :tSPACE, nil, [11, 12], + :tSTRING_DVAR, nil, [12, 13], + :tIVAR, '@a', [13, 15], + :tSPACE, nil, [15, 15], + :tSTRING_END, ']', [15, 16]) end def test_string_pct_I assert_scanned("%I(s1 s2)", - :tSYMBOLS_BEG, "%I(", - :tSTRING_CONTENT, "s1", - :tSPACE, nil, - :tSTRING_CONTENT, "s2", - :tSPACE, nil, - :tSTRING_END, ')') + :tSYMBOLS_BEG, "%I(", [0, 3], + :tSTRING_CONTENT, "s1", [3, 5], + :tSPACE, nil, [5, 6], + :tSTRING_CONTENT, "s2", [6, 8], + :tSPACE, nil, [8, 8], + :tSTRING_END, ')', [8, 9]) end def test_string_pct_angle assert_scanned("%", - :tSTRING_BEG, '%<', - :tSTRING_CONTENT, "blah", - :tSTRING_END, '>') + :tSTRING_BEG, '%<', [0, 2], + :tSTRING_CONTENT, "blah", [2, 6], + :tSTRING_END, '>', [6, 7]) end def test_string_pct_pct assert_scanned("%%blah%", - :tSTRING_BEG, '%%', - :tSTRING_CONTENT, "blah", - :tSTRING_END, '%') + :tSTRING_BEG, '%%', [0, 2], + :tSTRING_CONTENT, "blah", [2, 6], + :tSTRING_END, '%', [6, 7]) end def test_string_pct_w assert_scanned("%w[s1 s2 ]", - :tQWORDS_BEG, "%w[", - :tSTRING_CONTENT, "s1", - :tSPACE, nil, - :tSTRING_CONTENT, "s2", - :tSPACE, nil, - :tSTRING_END, "]") + :tQWORDS_BEG, "%w[", [0, 3], + :tSTRING_CONTENT, "s1", [3, 5], + :tSPACE, nil, [5, 6], + :tSTRING_CONTENT, "s2", [6, 8], + :tSPACE, nil, [8, 9], + :tSTRING_END, "]", [9, 10]) end def test_string_pct_w_incomplete refute_scanned("%w[s1 ", - :tQWORDS_BEG, "%w[", - :tSTRING_CONTENT, "s1", - :tSPACE, nil) + :tQWORDS_BEG, "%w[", [0, 3], + :tSTRING_CONTENT, "s1", [3, 5], + :tSPACE, nil, [5, 6]) end def test_string_pct_w_bs_nl assert_scanned("%w[s1 \\\ns2]", - :tQWORDS_BEG, "%w[", - :tSTRING_CONTENT, "s1", - :tSPACE, nil, - :tSTRING_CONTENT, "\ns2", - :tSPACE, nil, - :tSTRING_END, ']') + :tQWORDS_BEG, "%w[", [0, 3], + :tSTRING_CONTENT, "s1", [3, 5], + :tSPACE, nil, [5, 6], + :tSTRING_CONTENT, "\ns2", [6, 10], + :tSPACE, nil, [10, 10], + :tSTRING_END, ']', [10, 11]) end def test_string_pct_w_bs_sp assert_scanned("%w[s\\ 1 s\\ 2]", - :tQWORDS_BEG, "%w[", - :tSTRING_CONTENT, "s 1", - :tSPACE, nil, - :tSTRING_CONTENT, "s 2", - :tSPACE, nil, - :tSTRING_END, ']') + :tQWORDS_BEG, "%w[", [0, 3], + :tSTRING_CONTENT, "s 1", [3, 7], + :tSPACE, nil, [7, 8], + :tSTRING_CONTENT, "s 2", [8, 12], + :tSPACE, nil, [12, 12], + :tSTRING_END, ']', [12, 13]) end def test_string_pct_w_tab assert_scanned("%w[abc\tdef]", - :tQWORDS_BEG, "%w[", - :tSTRING_CONTENT, "abc", - :tSPACE, nil, - :tSTRING_CONTENT, "def", - :tSPACE, nil, - :tSTRING_END, ']') + :tQWORDS_BEG, "%w[", [0, 3], + :tSTRING_CONTENT, "abc", [3, 6], + :tSPACE, nil, [6, 7], + :tSTRING_CONTENT, "def", [7, 10], + :tSPACE, nil, [10, 10], + :tSTRING_END, ']', [10, 11]) end def test_string_pct_i assert_scanned("%i(s1 s2)", - :tQSYMBOLS_BEG, "%i(", - :tSTRING_CONTENT, "s1", - :tSPACE, nil, - :tSTRING_CONTENT, "s2", - :tSPACE, nil, - :tSTRING_END, ')') + :tQSYMBOLS_BEG, "%i(", [0, 3], + :tSTRING_CONTENT, "s1", [3, 5], + :tSPACE, nil, [5, 6], + :tSTRING_CONTENT, "s2", [6, 8], + :tSPACE, nil, [8, 8], + :tSTRING_END, ')', [8, 9]) end def test_string_pct_backslash assert_scanned("%\\a\\", - :tSTRING_BEG, "%\\", - :tSTRING_CONTENT, "a", - :tSTRING_END, "\\") + :tSTRING_BEG, "%\\", [0, 2], + :tSTRING_CONTENT, "a", [2, 3], + :tSTRING_END, "\\", [3, 4]) end def test_string_pct_backslash_with_bad_escape # No escapes are allowed in a backslash-delimited string refute_scanned("%\\a\\n\\", - :tSTRING_BEG, "%\\", - :tSTRING_CONTENT, "a", - :tSTRING_END, "\\", - :tIDENTIFIER, "n") + :tSTRING_BEG, "%\\", [0, 2], + :tSTRING_CONTENT, "a", [2, 3], + :tSTRING_END, "\\", [3, 4], + :tIDENTIFIER, "n", [4, 5]) end def test_string_pct_intertwined_with_heredoc assert_scanned("<<-foo + %\\a\nbar\nfoo\nb\\", - :tSTRING_BEG, "<<\"", - :tSTRING_CONTENT, "bar\n", - :tSTRING_END, "foo", - :tPLUS, "+", - :tSTRING_BEG, "%\\", - :tSTRING_CONTENT, "a\n", - :tSTRING_CONTENT, "b", - :tSTRING_END, "\\") + :tSTRING_BEG, "<<\"", [0, 6], + :tSTRING_CONTENT, "bar\n", [13, 17], + :tSTRING_END, "foo", [17, 20], + :tPLUS, "+", [7, 8], + :tSTRING_BEG, "%\\", [9, 11], + :tSTRING_CONTENT, "a\n", [11, 13], + :tSTRING_CONTENT, "b", [21, 22], + :tSTRING_END, "\\", [22, 23]) end def test_string_pct_q_backslash assert_scanned("%q\\a\\", - :tSTRING_BEG, "%q\\", - :tSTRING_CONTENT, "a", - :tSTRING_END, "\\") + :tSTRING_BEG, "%q\\", [0, 3], + :tSTRING_CONTENT, "a", [3, 4], + :tSTRING_END, "\\", [4, 5]) end def test_string_pct_Q_backslash assert_scanned("%Q\\a\\", - :tSTRING_BEG, "%Q\\", - :tSTRING_CONTENT, "a", - :tSTRING_END, "\\") + :tSTRING_BEG, "%Q\\", [0, 3], + :tSTRING_CONTENT, "a", [3, 4], + :tSTRING_END, "\\", [4, 5]) end def test_string_single assert_scanned("'string'", - :tSTRING, "string") + :tSTRING, "string", [0, 8]) end def test_string_single_escape_chars assert_scanned("'s\\tri\\ng'", - :tSTRING, "s\\tri\\ng") + :tSTRING, "s\\tri\\ng", [0, 10]) end def test_string_single_nl assert_scanned("'blah\\\nblah'", - :tSTRING_BEG, "'", - :tSTRING_CONTENT, "blah\\\n", - :tSTRING_CONTENT, "blah", - :tSTRING_END, "'") + :tSTRING_BEG, "'", [0, 1], + :tSTRING_CONTENT, "blah\\\n", [1, 7], + :tSTRING_CONTENT, "blah", [7, 11], + :tSTRING_END, "'", [11, 12]) end def test_symbol assert_scanned(":symbol", - :tSYMBOL, "symbol") + :tSYMBOL, "symbol", [0, 7]) end def test_symbol_double assert_scanned(":\"symbol\"", - :tSYMBEG, ":\"", - :tSTRING_CONTENT, "symbol", - :tSTRING_END, "\"") + :tSYMBEG, ":\"", [0, 2], + :tSTRING_CONTENT, "symbol", [2, 8], + :tSTRING_END, "\"", [8, 9]) end def test_symbol_single assert_scanned(":'symbol'", - :tSYMBEG, ":'", - :tSTRING_CONTENT, "symbol", - :tSTRING_END, "'") + :tSYMBEG, ":'", [0, 2], + :tSTRING_CONTENT, "symbol", [2, 8], + :tSTRING_END, "'", [8, 9]) end def test_ternary assert_scanned("a ? b : c", - :tIDENTIFIER, "a", - :tEH, "?", - :tIDENTIFIER, "b", - :tCOLON, ":", - :tIDENTIFIER, "c") + :tIDENTIFIER, "a", [0, 1], + :tEH, "?", [2, 3], + :tIDENTIFIER, "b", [4, 5], + :tCOLON, ":", [6, 7], + :tIDENTIFIER, "c", [8, 9]) assert_scanned("a ?b : c", - :tIDENTIFIER, "a", - :tINTEGER, 98, - :tCOLON, ":", - :tIDENTIFIER, "c") + :tIDENTIFIER, "a", [0, 1], + :tINTEGER, 98, [2, 4], + :tCOLON, ":", [5, 6], + :tIDENTIFIER, "c", [7, 8]) assert_scanned("a ?bb : c", # GAH! MATZ!!! - :tIDENTIFIER, "a", - :tEH, "?", - :tIDENTIFIER, "bb", - :tCOLON, ":", - :tIDENTIFIER, "c") + :tIDENTIFIER, "a", [0, 1], + :tEH, "?", [2, 3], + :tIDENTIFIER, "bb", [3, 5], + :tCOLON, ":", [6, 7], + :tIDENTIFIER, "c", [8, 9]) assert_scanned("42 ?", # 42 forces expr_end - :tINTEGER, 42, - :tEH, "?") + :tINTEGER, 42, [0, 2], + :tEH, "?", [3, 4]) end def test_tilde - assert_scanned "~", :tTILDE, "~" + assert_scanned "~", :tTILDE, "~", [0, 1] end def test_tilde_unary @lex.state = :expr_fname - assert_scanned "~@", :tTILDE, "~@" + assert_scanned "~@", :tTILDE, "~@", [0, 2] end def test_uminus assert_scanned("-blah", - :tUMINUS, "-", - :tIDENTIFIER, "blah") + :tUMINUS, "-", [0, 1], + :tIDENTIFIER, "blah", [1, 5]) end def test_underscore - assert_scanned("_var", :tIDENTIFIER, "_var") + assert_scanned("_var", :tIDENTIFIER, "_var", [0, 4]) end def test_underscore_end assert_scanned("__END__\n") assert_scanned("__END__") assert_scanned("__END__ foo", - :tIDENTIFIER, '__END__', - :tIDENTIFIER, 'foo') + :tIDENTIFIER, '__END__', [0, 7], + :tIDENTIFIER, 'foo', [8, 11]) assert_scanned("__END__\rfoo", - :tIDENTIFIER, '__END__', - :tIDENTIFIER, 'foo') + :tIDENTIFIER, '__END__', [0, 7], + :tIDENTIFIER, 'foo', [8, 11]) end def test_uplus assert_scanned("+blah", - :tUPLUS, "+", - :tIDENTIFIER, "blah") + :tUPLUS, "+", [0, 1], + :tIDENTIFIER, "blah", [1, 5]) end def test_if_unless_mod assert_scanned("return if true unless false", - :kRETURN, "return", - :kIF_MOD, "if", - :kTRUE, "true", - :kUNLESS_MOD, "unless", - :kFALSE, "false") + :kRETURN, "return", [0, 6], + :kIF_MOD, "if", [7, 9], + :kTRUE, "true", [10, 14], + :kUNLESS_MOD, "unless", [15, 21], + :kFALSE, "false", [22, 27]) end def test_if_stmt assert_scanned("if true\n return end", - :kIF, "if", - :kTRUE, "true", - :tNL, nil, - :kRETURN, "return", - :kEND, "end") + :kIF, "if", [0, 2], + :kTRUE, "true", [3, 7], + :tNL, nil, [7, 8], + :kRETURN, "return", [9, 15], + :kEND, "end", [16, 19]) end def test_sclass_label setup_lexer 20 assert_scanned("class << a:b", - :kCLASS, 'class', - :tLSHFT, '<<', - :tIDENTIFIER, 'a', - :tSYMBOL, 'b') + :kCLASS, 'class', [0, 5], + :tLSHFT, '<<', [6, 8], + :tIDENTIFIER, 'a', [9, 10], + :tSYMBOL, 'b', [10, 12]) end def test_static_env @@ -2350,10 +2355,10 @@ def test_static_env @lex.static_env = env assert_scanned("a [42]", - :tIDENTIFIER, "a", - :tLBRACK2, "[", - :tINTEGER, 42, - :tRBRACK, "]") + :tIDENTIFIER, "a", [0, 1], + :tLBRACK2, "[", [2, 3], + :tINTEGER, 42, [3, 5], + :tRBRACK, "]", [5, 6]) end def test_int_suffix @@ -2361,19 +2366,19 @@ def test_int_suffix setup_lexer version assert_scanned("42r", - :tINTEGER, 42, - :tIDENTIFIER, 'r') + :tINTEGER, 42, [0, 2], + :tIDENTIFIER, 'r', [2, 3]) assert_scanned("42if", - :tINTEGER, 42, - :kIF_MOD, 'if') + :tINTEGER, 42, [0, 2], + :kIF_MOD, 'if', [2, 4]) end setup_lexer 21 - assert_scanned("42r", :tRATIONAL, Rational(42)) - assert_scanned("42i", :tIMAGINARY, Complex(0, 42)) - assert_scanned("42ri", :tIMAGINARY, Complex(0, Rational(42))) + assert_scanned("42r", :tRATIONAL, Rational(42), [0, 3]) + assert_scanned("42i", :tIMAGINARY, Complex(0, 42), [0, 3]) + assert_scanned("42ri", :tIMAGINARY, Complex(0, Rational(42)), [0, 4]) end def test_float_suffix @@ -2381,16 +2386,16 @@ def test_float_suffix setup_lexer version assert_scanned("42.1r", - :tFLOAT, 42.1, - :tIDENTIFIER, 'r') + :tFLOAT, 42.1, [0, 4], + :tIDENTIFIER, 'r', [4, 5]) assert_scanned("42.1if", - :tFLOAT, 42.1, - :kIF_MOD, 'if') + :tFLOAT, 42.1, [0, 4], + :kIF_MOD, 'if', [4, 6]) assert_scanned("1e1r", - :tFLOAT, 1e1, - :tIDENTIFIER, 'r') + :tFLOAT, 1e1, [0, 3], + :tIDENTIFIER, 'r', [3, 4]) end begin @@ -2399,23 +2404,23 @@ def test_float_suffix setup_lexer 21 - assert_scanned("42.1r", :tRATIONAL, Rational(421, 10)) - assert_scanned("42.1i", :tIMAGINARY, Complex(0, 42.1)) - assert_scanned("42.1ri", :tIMAGINARY, Complex(0, Rational(421, 10))) + assert_scanned("42.1r", :tRATIONAL, Rational(421, 10), [0, 5]) + assert_scanned("42.1i", :tIMAGINARY, Complex(0, 42.1), [0, 5]) + assert_scanned("42.1ri", :tIMAGINARY, Complex(0, Rational(421, 10)), [0, 6]) assert_scanned("42.1ir", - :tIMAGINARY, Complex(0, 42.1), - :tIDENTIFIER, 'r') + :tIMAGINARY, Complex(0, 42.1), [0, 5], + :tIDENTIFIER, 'r', [5, 6]) - assert_scanned("1e1i", :tIMAGINARY, Complex(0, 1e1)) + assert_scanned("1e1i", :tIMAGINARY, Complex(0, 1e1), [0, 4]) assert_scanned("1e1r", - :tFLOAT, 1e1, - :tIDENTIFIER, 'r') + :tFLOAT, 1e1, [0, 3], + :tIDENTIFIER, 'r', [3, 4]) assert_scanned("1e1ri", - :tFLOAT, 1e1, - :tIDENTIFIER, 'ri') + :tFLOAT, 1e1, [0, 3], + :tIDENTIFIER, 'ri', [3, 5]) assert_scanned("1e1ir", - :tIMAGINARY, Complex(0, 1e1), - :tIDENTIFIER, 'r') + :tIMAGINARY, Complex(0, 1e1), [0, 4], + :tIDENTIFIER, 'r', [4, 5]) rescue NoMethodError # Ruby not modern enough end @@ -2427,26 +2432,26 @@ def test_float_suffix def test_fluent_dot assert_scanned("x\n.y", - :tIDENTIFIER, 'x', - :tDOT, '.', - :tIDENTIFIER, 'y') + :tIDENTIFIER, 'x', [0, 1], + :tDOT, '.', [2, 3], + :tIDENTIFIER, 'y', [3, 4]) assert_scanned("x\n .y", - :tIDENTIFIER, 'x', - :tDOT, '.', - :tIDENTIFIER, 'y') + :tIDENTIFIER, 'x', [0, 1], + :tDOT, '.', [4, 5], + :tIDENTIFIER, 'y', [5, 6]) assert_scanned("x # comment\n .y", - :tIDENTIFIER, 'x', - :tDOT, '.', - :tIDENTIFIER, 'y') + :tIDENTIFIER, 'x', [0, 1], + :tDOT, '.', [14, 15], + :tIDENTIFIER, 'y', [15, 16]) end def test_fluent_and_dot assert_scanned("x\n&.y", - :tIDENTIFIER, 'x', - :tANDDOT, '&.', - :tIDENTIFIER, 'y') + :tIDENTIFIER, 'x', [0, 1], + :tANDDOT, '&.', [2, 4], + :tIDENTIFIER, 'y', [4, 5]) end # @@ -2456,23 +2461,23 @@ def test_fluent_and_dot def test_whitespace_fname @lex.state = :expr_fname assert_scanned('class', - :kCLASS, 'class') + :kCLASS, 'class', [0, 5]) @lex.state = :expr_fname assert_scanned(' class', - :kCLASS, 'class') + :kCLASS, 'class', [1, 6]) @lex.state = :expr_fname assert_scanned("\nclass", - :kCLASS, 'class') + :kCLASS, 'class', [1, 6]) @lex.state = :expr_fname assert_scanned("\\\nclass", - :kCLASS, 'class') + :kCLASS, 'class', [2, 7]) @lex.state = :expr_fname assert_scanned("#foo\nclass", - :kCLASS, 'class') + :kCLASS, 'class', [5, 10]) end def test_whitespace_endfn @@ -2480,159 +2485,159 @@ def test_whitespace_endfn @lex.state = :expr_endfn assert_scanned('foo:', - :tLABEL, 'foo') + :tLABEL, 'foo', [0, 4]) @lex.state = :expr_endfn assert_scanned(' foo:', - :tLABEL, 'foo') + :tLABEL, 'foo', [1, 5]) @lex.state = :expr_endfn assert_scanned("\nfoo:", - :tNL, nil, - :tIDENTIFIER, 'foo', - :tCOLON, ':') + :tNL, nil, [0, 1], + :tIDENTIFIER, 'foo', [1, 4], + :tCOLON, ':', [4, 5]) @lex.state = :expr_endfn assert_scanned("\nfoo: ", - :tNL, nil, - :tIDENTIFIER, 'foo', - :tCOLON, ':') + :tNL, nil, [0, 1], + :tIDENTIFIER, 'foo', [1, 4], + :tCOLON, ':', [4, 5]) @lex.state = :expr_endfn assert_scanned("\\\nfoo:", - :tLABEL, 'foo') + :tLABEL, 'foo', [2, 6]) @lex.state = :expr_endfn assert_scanned("#foo\nfoo:", - :tNL, nil, - :tIDENTIFIER, 'foo', - :tCOLON, ':') + :tNL, nil, [4, 5], + :tIDENTIFIER, 'foo', [5, 8], + :tCOLON, ':', [8, 9]) @lex.state = :expr_endfn assert_scanned("#foo\nfoo: ", - :tNL, nil, - :tIDENTIFIER, 'foo', - :tCOLON, ':') + :tNL, nil, [4, 5], + :tIDENTIFIER, 'foo', [5, 8], + :tCOLON, ':', [8, 9]) end def test_whitespace_dot @lex.state = :expr_dot assert_scanned('class', - :tIDENTIFIER, 'class') + :tIDENTIFIER, 'class', [0, 5]) @lex.state = :expr_dot assert_scanned(' class', - :tIDENTIFIER, 'class') + :tIDENTIFIER, 'class', [1, 6]) @lex.state = :expr_dot assert_scanned("\nclass", - :tIDENTIFIER, 'class') + :tIDENTIFIER, 'class', [1, 6]) @lex.state = :expr_dot assert_scanned("\\\nclass", - :tIDENTIFIER, 'class') + :tIDENTIFIER, 'class', [2, 7]) @lex.state = :expr_dot assert_scanned("#foo\nclass", - :tIDENTIFIER, 'class') + :tIDENTIFIER, 'class', [5, 10]) end def test_whitespace_arg @lex.state = :expr_arg assert_scanned('+', - :tPLUS, '+') + :tPLUS, '+', [0, 1]) @lex.state = :expr_arg assert_scanned(' +', - :tUPLUS, '+') + :tUPLUS, '+', [1, 2]) @lex.state = :expr_arg assert_scanned("\n+", - :tNL, nil, - :tUPLUS, '+') + :tNL, nil, [0, 1], + :tUPLUS, '+', [1, 2]) @lex.state = :expr_arg assert_scanned("\\\n+", - :tUPLUS, '+') + :tUPLUS, '+', [2, 3]) @lex.state = :expr_arg assert_scanned("\\\n +", - :tUPLUS, '+') + :tUPLUS, '+', [3, 4]) @lex.state = :expr_arg assert_scanned("#foo\n+", - :tNL, nil, - :tUPLUS, '+') + :tNL, nil, [4, 5], + :tUPLUS, '+', [5, 6]) end def test_whitespace_endarg @lex.state = :expr_endarg assert_scanned('{', - :tLBRACE_ARG, '{') + :tLBRACE_ARG, '{', [0, 1]) @lex.state = :expr_endarg assert_scanned(' {', - :tLBRACE_ARG, '{') + :tLBRACE_ARG, '{', [1, 2]) @lex.state = :expr_endarg assert_scanned("\n{", - :tNL, nil, - :tLBRACE, '{') + :tNL, nil, [0, 1], + :tLBRACE, '{', [1, 2]) @lex.state = :expr_endarg assert_scanned("\\\n{", - :tLBRACE_ARG, '{') + :tLBRACE_ARG, '{', [2, 3]) @lex.state = :expr_endarg assert_scanned("#foo\n{", - :tNL, nil, - :tLBRACE, '{') + :tNL, nil, [4, 5], + :tLBRACE, '{', [5, 6]) end def test_whitespace_mid @lex.state = :expr_mid assert_scanned('+', - :tUPLUS, '+') + :tUPLUS, '+', [0, 1]) @lex.state = :expr_mid assert_scanned(' +', - :tUPLUS, '+') + :tUPLUS, '+', [1, 2]) @lex.state = :expr_mid assert_scanned("\n+", - :tNL, nil, - :tUPLUS, '+') + :tNL, nil, [0, 1], + :tUPLUS, '+', [1, 2]) @lex.state = :expr_mid assert_scanned("\\\n+", - :tUPLUS, '+') + :tUPLUS, '+', [2, 3]) @lex.state = :expr_mid assert_scanned("#foo\n+", - :tNL, nil, - :tUPLUS, '+') + :tNL, nil, [4, 5], + :tUPLUS, '+', [5, 6]) end def test_whitespace_beg @lex.state = :expr_beg assert_scanned('+', - :tUPLUS, '+') + :tUPLUS, '+', [0, 1]) @lex.state = :expr_beg assert_scanned(' +', - :tUPLUS, '+') + :tUPLUS, '+', [1, 2]) @lex.state = :expr_beg assert_scanned("\n+", - :tUPLUS, '+') + :tUPLUS, '+', [1, 2]) @lex.state = :expr_beg assert_scanned("\\\n+", - :tUPLUS, '+') + :tUPLUS, '+', [2, 3]) @lex.state = :expr_beg assert_scanned("#foo\n+", - :tUPLUS, '+') + :tUPLUS, '+', [5, 6]) end def test_whitespace_value @@ -2640,71 +2645,71 @@ def test_whitespace_value @lex.state = :expr_value assert_scanned('a:b', - :tIDENTIFIER, 'a', - :tSYMBOL, 'b') + :tIDENTIFIER, 'a', [0, 1], + :tSYMBOL, 'b', [1, 3]) @lex.state = :expr_value assert_scanned(' a:b', - :tIDENTIFIER, 'a', - :tSYMBOL, 'b') + :tIDENTIFIER, 'a', [1, 2], + :tSYMBOL, 'b', [2, 4]) @lex.state = :expr_value assert_scanned("\na:b", - :tIDENTIFIER, 'a', - :tSYMBOL, 'b') + :tIDENTIFIER, 'a', [1, 2], + :tSYMBOL, 'b', [2, 4]) @lex.state = :expr_value assert_scanned("\\\na:b", - :tIDENTIFIER, 'a', - :tSYMBOL, 'b') + :tIDENTIFIER, 'a', [2, 3], + :tSYMBOL, 'b', [3, 5]) @lex.state = :expr_value assert_scanned("#foo\na:b", - :tIDENTIFIER, 'a', - :tSYMBOL, 'b') + :tIDENTIFIER, 'a', [5, 6], + :tSYMBOL, 'b', [6, 8]) end def test_whitespace_end @lex.state = :expr_end assert_scanned('+ 1', - :tPLUS, '+', - :tINTEGER, 1) + :tPLUS, '+', [0, 1], + :tINTEGER, 1, [2, 3]) @lex.state = :expr_end assert_scanned(' + 1', - :tPLUS, '+', - :tINTEGER, 1) + :tPLUS, '+', [1, 2], + :tINTEGER, 1, [3, 4]) @lex.state = :expr_end assert_scanned("\n+ 1", - :tNL, nil, - :tUPLUS, '+', - :tINTEGER, 1) + :tNL, nil, [0, 1], + :tUPLUS, '+', [1, 2], + :tINTEGER, 1, [3, 4]) @lex.state = :expr_end assert_scanned("\\\n+ 1", - :tPLUS, '+', - :tINTEGER, 1) + :tPLUS, '+', [2, 3], + :tINTEGER, 1, [4, 5]) @lex.state = :expr_end assert_scanned("#foo\n+ 1", - :tNL, nil, - :tUPLUS, '+', - :tINTEGER, 1) + :tNL, nil, [4, 5], + :tUPLUS, '+', [5, 6], + :tINTEGER, 1, [7, 8]) end def test_whitespace_cr setup_lexer(20) assert_scanned("<", - :kRESCUE, 'rescue', - :tASSOC, '=>') + :kRESCUE, 'rescue', [0, 6], + :tASSOC, '=>', [6, 8]) end def test_bug_expr_arg_percent @lex.state = :expr_arg assert_scanned("%[", - :tPERCENT, "%", - :tLBRACK, "[") + :tPERCENT, "%", [0, 1], + :tLBRACK, "[", [1, 2]) @lex.state = :expr_arg assert_scanned("%=1", - :tOP_ASGN, "%", - :tINTEGER, 1) + :tOP_ASGN, "%", [0, 2], + :tINTEGER, 1, [2, 3]) @lex.state = :expr_arg assert_scanned(" %[1]", - :tSTRING_BEG, "%[", - :tSTRING_CONTENT, '1', - :tSTRING_END, ']') + :tSTRING_BEG, "%[", [1, 3], + :tSTRING_CONTENT, '1', [3, 4], + :tSTRING_END, ']', [4, 5]) @lex.state = :expr_arg assert_scanned(" %=1=", - :tOP_ASGN, "%", - :tINTEGER, 1, - :tEQL, "=") + :tOP_ASGN, "%", [1, 3], + :tINTEGER, 1, [3, 4], + :tEQL, "=", [4, 5]) @lex.state = :expr_arg assert_scanned(" %\n", - :tPERCENT, '%') + :tPERCENT, '%', [1, 2]) end def test_bug_expr_arg_lt_lt @lex.state = :expr_arg assert_scanned("<