From 203283e8d33aeaa04484c1908be061d8856db699 Mon Sep 17 00:00:00 2001 From: Ryan Davis Date: Sun, 19 Oct 2008 19:19:19 -0800 Subject: [PATCH] More cleanup, switching string tokens to named tokens... and stuff [git-p4: depot-paths = "//src/ruby_parser/dev/": change = 4376] --- bin/ruby_parse | 1 - lib/ruby_lexer.rb | 22 +-- lib/ruby_parser.y | 413 ++++++++++++++++++++-------------------- test/test_ruby_lexer.rb | 226 +++++++++++----------- 4 files changed, 330 insertions(+), 332 deletions(-) diff --git a/bin/ruby_parse b/bin/ruby_parse index 36739c0e..c4329557 100755 --- a/bin/ruby_parse +++ b/bin/ruby_parse @@ -5,7 +5,6 @@ $g ||= false require 'rubygems' require 'ruby_parser' -require 'timeout' require 'pp' good = bad = 0 diff --git a/lib/ruby_lexer.rb b/lib/ruby_lexer.rb index 90934e07..01682b4a 100644 --- a/lib/ruby_lexer.rb +++ b/lib/ruby_lexer.rb @@ -384,7 +384,7 @@ def parse_string(quote) # 65 lines if awords then quote[1] = nil # @@stats[:def_parse_string_return_awords] += 1 if SPY - return ' ' + return :tSPACE elsif regexp then self.yacc_value = self.regx_options self.lineno = nil @@ -399,7 +399,7 @@ def parse_string(quote) # 65 lines if space then # @@stats[:def_parse_string_return_space] += 1 if SPY - return ' ' + return :tSPACE end self.string_buffer = [] @@ -745,7 +745,7 @@ def yylex # 826 lines self.command_start = true self.lex_state = :expr_beg - return "\n" + return :tNL elsif src.scan(/[\]\)\}]/) then # @@stats[:case3] += 1 if SPY cond.lexpop @@ -782,7 +782,7 @@ def yylex # 826 lines # @@stats[:case8] += 1 if SPY self.lex_state = :expr_beg self.yacc_value = "," - return src.matched + return :tCOMMA elsif src.scan(/\(/) then # @@stats[:case9] += 1 if SPY result = :tLPAREN2 @@ -838,7 +838,7 @@ def yylex # 826 lines else self.fix_arg_lex_state self.yacc_value = '=' - return '=' + return :tEQL end end elsif src.scan(/\"(#{ESC_RE}|#(#{ESC_RE}|[^\{\#\@\$\"\\])|[^\"\\\#])*\"/o) then @@ -884,7 +884,7 @@ def yylex # 826 lines src.check(/\s/)) then self.lex_state = :expr_beg self.yacc_value = ":" - return ':' + return :tCOLON end case @@ -1144,7 +1144,7 @@ def yylex # 826 lines if lex_state == :expr_end || lex_state == :expr_endarg then self.lex_state = :expr_beg self.yacc_value = "?" - return '?' + return :tEH end if src.eos? then @@ -1168,11 +1168,11 @@ def yylex # 826 lines # ternary self.lex_state = :expr_beg self.yacc_value = "?" - return '?' + return :tEH elsif src.check(/\w(?=\w)/) then # ternary, also self.lex_state = :expr_beg self.yacc_value = "?" - return '?' + return :tEH end c = if src.scan(/\\/) then @@ -1257,7 +1257,7 @@ def yylex # 826 lines self.command_start = true self.lex_state = :expr_beg self.yacc_value = ";" - return src.matched + return :tSEMI elsif src.scan(/\~/) then # @@stats[:case54] += 1 if SPY if lex_state == :expr_fname || lex_state == :expr_dot then @@ -1343,7 +1343,7 @@ def yylex # 826 lines # @@stats[:case63] += 1 if SPY self.lex_state = :expr_end self.yacc_value = "$" - return '$' + return "$" elsif src.scan(/\$\w+/) # @@stats[:case64] += 1 if SPY self.lex_state = :expr_end diff --git a/lib/ruby_parser.y b/lib/ruby_parser.y index 6a893d5e..e155a599 100644 --- a/lib/ruby_parser.y +++ b/lib/ruby_parser.y @@ -16,7 +16,7 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS tTILDE tPERCENT tDIVIDE tPLUS tMINUS tLT tGT tPIPE tBANG tCARET tLCURLY tRCURLY tBACK_REF2 tSYMBEG tSTRING_BEG tXSTRING_BEG tREGEXP_BEG tWORDS_BEG tAWORDS_BEG tSTRING_DBEG tSTRING_DVAR tSTRING_END tSTRING - tSYMBOL tLAST_TOKEN + tSYMBOL tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAST_TOKEN prechigh right tBANG tTILDE tUPLUS @@ -32,9 +32,9 @@ prechigh left tANDOP left tOROP nonassoc tDOT2 tDOT3 - right '?' ':' + right tEH tCOLON left kRESCUE_MOD - right '=' tOP_ASGN + right tEQL tOP_ASGN nonassoc kDEFINED right kNOT left kOR kAND @@ -67,11 +67,11 @@ rule | stmt | stmts terms stmt { - result = self.block_append(val[0], val[2]) + result = self.block_append val[0], val[2] } | error stmt { - result = val[1]; + result = val[1] } stmt: kALIAS fitem @@ -93,7 +93,7 @@ rule } | kALIAS tGVAR tNTH_REF { - yyerror("can't make alias for the number variables"); + yyerror "can't make alias for the number variables" } | kUNDEF undef_list { @@ -122,9 +122,9 @@ rule | klBEGIN { if (self.in_def || self.in_single > 0) then - # yyerror("BEGIN in method"); HACK + yyerror "BEGIN in method" end - self.env.extend; + self.env.extend } tLCURLY compstmt tRCURLY { @@ -134,15 +134,15 @@ rule | klEND tLCURLY compstmt tRCURLY { if (self.in_def || self.in_single > 0) then - # yyerror("END in method; use at_exit"); HACK + yyerror "END in method; use at_exit" end result = new_iter s(:postexe), nil, val[2] } - | lhs '=' command_call + | lhs tEQL command_call { - result = self.node_assign(val[0], val[2]) + result = self.node_assign val[0], val[2] } - | mlhs '=' command_call + | mlhs tEQL command_call { result = new_masgn val[0], val[2], :wrap } @@ -150,35 +150,35 @@ rule { result = new_op_asgn val } - | primary_value '[' aref_args tRBRACK tOP_ASGN command_call + | primary_value "[" aref_args tRBRACK tOP_ASGN command_call { result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5]) } | primary_value tDOT tIDENTIFIER tOP_ASGN command_call { - result = s(:op_asgn, val[0], val[4], val[2], val[3]); + result = s(:op_asgn, val[0], val[4], val[2], val[3]) } | primary_value tDOT tCONSTANT tOP_ASGN command_call { - result = s(:op_asgn, val[0], val[4], val[2], val[3]); + result = s(:op_asgn, val[0], val[4], val[2], val[3]) } | primary_value tCOLON2 tIDENTIFIER tOP_ASGN command_call { - result = s(:op_asgn, val[0], val[4], val[2], val[3]); + result = s(:op_asgn, val[0], val[4], val[2], val[3]) } | backref tOP_ASGN command_call { - self.backref_assign_error(val[0]); + self.backref_assign_error val[0] } - | lhs '=' mrhs + | lhs tEQL mrhs { - result = self.node_assign(val[0], s(:svalue, val[2])) + result = self.node_assign val[0], s(:svalue, val[2]) } - | mlhs '=' arg_value + | mlhs tEQL arg_value { result = new_masgn val[0], val[2], :wrap } - | mlhs '=' mrhs + | mlhs tEQL mrhs { result = new_masgn val[0], val[2] } @@ -298,26 +298,26 @@ rule mlhs: mlhs_basic | tLPAREN mlhs_entry tRPAREN { - result = val[1]; + result = val[1] } mlhs_entry: mlhs_basic | tLPAREN mlhs_entry tRPAREN { - result = s(:masgn, s(:array, val[1])); + result = s(:masgn, s(:array, val[1])) } mlhs_basic: mlhs_head { - result = s(:masgn, val[0]); + result = s(:masgn, val[0]) } | mlhs_head mlhs_item { - result = s(:masgn, val[0] << val[1].compact); + result = s(:masgn, val[0] << val[1].compact) } | mlhs_head tSTAR mlhs_node { - result = s(:masgn, val[0] << s(:splat, val[2])); + result = s(:masgn, val[0] << s(:splat, val[2])) } | mlhs_head tSTAR { @@ -335,42 +335,42 @@ rule mlhs_item: mlhs_node | tLPAREN mlhs_entry tRPAREN { - result = val[1]; + result = val[1] } - mlhs_head: mlhs_item ',' + mlhs_head: mlhs_item tCOMMA { result = s(:array, val[0]) } - | mlhs_head mlhs_item ',' + | mlhs_head mlhs_item tCOMMA { result = val[0] << val[1].compact } mlhs_node: variable { - result = self.assignable(val[0]) + result = self.assignable val[0] } - | primary_value '[' aref_args tRBRACK + | primary_value "[" aref_args tRBRACK { - result = self.aryset(val[0], val[2]); + result = self.aryset val[0], val[2] } | primary_value tDOT tIDENTIFIER { - result = s(:attrasgn, val[0], :"#{val[2]}=", s(:arglist)); + result = s(:attrasgn, val[0], :"#{val[2]}=", s(:arglist)) } | primary_value tCOLON2 tIDENTIFIER { - result = s(:attrasgn, val[0], :"#{val[2]}=", s(:arglist)); + result = s(:attrasgn, val[0], :"#{val[2]}=", s(:arglist)) } | primary_value tDOT tCONSTANT { - result = s(:attrasgn, val[0], :"#{val[2]}=", s(:arglist)); + result = s(:attrasgn, val[0], :"#{val[2]}=", s(:arglist)) } | primary_value tCOLON2 tCONSTANT { if (self.in_def || self.in_single > 0) then - yyerror("dynamic constant assignment"); + yyerror "dynamic constant assignment" end result = s(:const, s(:colon2, val[0], val[2].to_sym), nil) @@ -378,23 +378,23 @@ rule | tCOLON3 tCONSTANT { if (self.in_def || self.in_single > 0) then - yyerror("dynamic constant assignment"); + yyerror "dynamic constant assignment" end result = s(:const, nil, s(:colon3, val[1].to_sym)) } | backref { - self.backref_assign_error(val[0]); + self.backref_assign_error val[0] } lhs: variable { - result = self.assignable(val[0]) + result = self.assignable val[0] } - | primary_value '[' aref_args tRBRACK + | primary_value "[" aref_args tRBRACK { - result = self.aryset(val[0], val[2]); + result = self.aryset val[0], val[2] } | primary_value tDOT tIDENTIFIER { @@ -411,7 +411,7 @@ rule | primary_value tCOLON2 tCONSTANT { if (self.in_def || self.in_single > 0) then - yyerror("dynamic constant assignment") + yyerror "dynamic constant assignment" end result = s(:const, s(:colon2, val[0], val[2].to_sym)) @@ -419,19 +419,19 @@ rule | tCOLON3 tCONSTANT { if (self.in_def || self.in_single > 0) then - yyerror("dynamic constant assignment"); + yyerror "dynamic constant assignment" end result = s(:const, s(:colon3, val[1].to_sym)) } | backref { - self.backref_assign_error(val[0]); + self.backref_assign_error val[0] } cname: tIDENTIFIER { - yyerror("class/module name must be CONSTANT"); + yyerror "class/module name must be CONSTANT" } | tCONSTANT @@ -445,20 +445,20 @@ rule } | primary_value tCOLON2 cname { - result = s(:colon2, val[0], val[2].to_sym); + result = s(:colon2, val[0], val[2].to_sym) } fname: tIDENTIFIER | tCONSTANT | tFID | op { lexer.lex_state = :expr_end - result = val[0]; + result = val[0] } | reswords { lexer.lex_state = :expr_end - result = val[0]; + result = val[0] } # TODO: cruby has fsym and dsym @@ -470,7 +470,7 @@ rule result = new_undef val[0] } | - undef_list ',' + undef_list tCOMMA { lexer.lex_state = :expr_fname } @@ -493,28 +493,27 @@ rule | kSUPER | kTHEN | kTRUE | kUNDEF | kWHEN | kYIELD | kIF_MOD | kUNLESS_MOD | kWHILE_MOD | kUNTIL_MOD | kRESCUE_MOD - arg: lhs '=' arg + arg: lhs tEQL arg { - result = self.node_assign(val[0], val[2]) + result = self.node_assign val[0], val[2] } - | lhs '=' arg kRESCUE_MOD arg + | lhs tEQL arg kRESCUE_MOD arg { - result = self.node_assign(val[0], - s(:rescue, val[2], s(:resbody, s(:array), val[4]))) + result = self.node_assign val[0], s(:rescue, val[2], s(:resbody, s(:array), val[4])) # result.line = val[0].line } | var_lhs tOP_ASGN arg { result = new_op_asgn val } - | primary_value '[' aref_args tRBRACK tOP_ASGN arg + | primary_value "[" aref_args tRBRACK tOP_ASGN arg { - result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5]); + result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5]) val[2][0] = :arglist } | primary_value tDOT tIDENTIFIER tOP_ASGN arg { - result = s(:op_asgn2, val[0], :"#{val[2]}=", val[3].to_sym, val[4]); + result = s(:op_asgn2, val[0], :"#{val[2]}=", val[3].to_sym, val[4]) } | primary_value tDOT tCONSTANT tOP_ASGN arg { @@ -522,20 +521,20 @@ rule } | primary_value tCOLON2 tIDENTIFIER tOP_ASGN arg { - result = s(:op_asgn, val[0], val[4], val[2], val[3]); + result = s(:op_asgn, val[0], val[4], val[2], val[3]) } | primary_value tCOLON2 tCONSTANT tOP_ASGN arg { - yyerror("constant re-assignment"); + yyerror "constant re-assignment" } | tCOLON3 tCONSTANT tOP_ASGN arg { - yyerror("constant re-assignment"); + yyerror "constant re-assignment" } | backref tOP_ASGN arg { - self.backref_assign_error(val[0]); + self.backref_assign_error val[0] } | arg tDOT2 arg { @@ -585,7 +584,7 @@ rule } | tUMINUS_NUM tFLOAT tPOW arg { - result = new_call(new_call(s(:lit, val[1]), :"**", s(:arglist, val[3])), :"-@", s(:arglist)); + result = new_call(new_call(s(:lit, val[1]), :"**", s(:arglist, val[3])), :"-@", s(:arglist)) } | tUPLUS arg { @@ -647,7 +646,7 @@ rule } | arg tMATCH arg { - result = self.get_match_node(val[0], val[2]) + result = self.get_match_node val[0], val[2] } | arg tNMATCH arg { @@ -684,11 +683,11 @@ rule } | kDEFINED opt_nl arg { - result = s(:defined, val[2]); + result = s(:defined, val[2]) } - | arg '?' arg ':' arg + | arg tEH arg tCOLON arg { - result = s(:if, val[0], val[2], val[4]); + result = s(:if, val[0], val[2], val[4]) } | primary @@ -700,16 +699,16 @@ rule aref_args: none | command opt_nl { - warning("parenthesize argument(s) for future version"); - result = s(:array, val[0]); + warning 'parenthesize argument(s) for future version' + result = s(:array, val[0]) } | args trailer { - result = val[0]; + result = val[0] } - | args ',' tSTAR arg opt_nl + | args tCOMMA tSTAR arg opt_nl { - result = self.arg_concat(val[0], val[3]) + result = self.arg_concat val[0], val[3] } | assocs trailer { @@ -727,17 +726,17 @@ rule } | tLPAREN2 call_args opt_nl tRPAREN { - result = val[1]; + result = val[1] } | tLPAREN2 block_call opt_nl tRPAREN { - warning("parenthesize argument(s) for future version"); - result = s(:array, val[1]); + warning "parenthesize argument(s) for future version" + result = s(:array, val[1]) } - | tLPAREN2 args ',' block_call opt_nl tRPAREN + | tLPAREN2 args tCOMMA block_call opt_nl tRPAREN { - warning("parenthesize argument(s) for future version"); - result = val[1].add(val[3]); + warning "parenthesize argument(s) for future version" + result = val[1].add val[3] } opt_paren_args: none @@ -745,97 +744,97 @@ rule call_args: command { - warning("parenthesize argument(s) for future version"); + warning "parenthesize argument(s) for future version" result = s(:array, val[0]) } | args opt_block_arg { - result = self.arg_blk_pass(val[0], val[1]); + result = self.arg_blk_pass val[0], val[1] } - | args ',' tSTAR arg_value opt_block_arg + | args tCOMMA tSTAR arg_value opt_block_arg { - result = self.arg_concat(val[0], val[3]); - result = self.arg_blk_pass(result, val[4]); + result = self.arg_concat val[0], val[3] + result = self.arg_blk_pass result, val[4] } | assocs opt_block_arg { result = s(:array, s(:hash, *val[0].values)) - result = self.arg_blk_pass(result, val[1]); + result = self.arg_blk_pass result, val[1] } - | assocs ',' tSTAR arg_value opt_block_arg + | assocs tCOMMA tSTAR arg_value opt_block_arg { - result = self.arg_concat(s(:array, s(:hash, *val[0].values)), val[3]) - result = self.arg_blk_pass(result, val[4]); + result = self.arg_concat s(:array, s(:hash, *val[0].values)), val[3] + result = self.arg_blk_pass result, val[4] } - | args ',' assocs opt_block_arg + | args tCOMMA assocs opt_block_arg { result = val[0] << s(:hash, *val[2].values) - result = self.arg_blk_pass(result, val[3]); + result = self.arg_blk_pass result, val[3] } - | args ',' assocs ',' tSTAR arg opt_block_arg + | args tCOMMA assocs tCOMMA tSTAR arg opt_block_arg { val[0] << s(:hash, *val[2].values) - result = self.arg_concat(val[0], val[5]) - result = self.arg_blk_pass(result, val[6]); + result = self.arg_concat val[0], val[5] + result = self.arg_blk_pass result, val[6] } | tSTAR arg_value opt_block_arg { - result = self.arg_blk_pass(s(:splat, val[1]), val[2]) + result = self.arg_blk_pass s(:splat, val[1]), val[2] } | block_arg - call_args2: arg_value ',' args opt_block_arg + call_args2: arg_value tCOMMA args opt_block_arg { - args = self.list_prepend(val[0], val[2]) - result = self.arg_blk_pass(args, val[3]) + args = self.list_prepend val[0], val[2] + result = self.arg_blk_pass args, val[3] } - | arg_value ',' block_arg + | arg_value tCOMMA block_arg { - result = self.arg_blk_pass(val[0], val[2]); + result = self.arg_blk_pass val[0], val[2] } - | arg_value ',' tSTAR arg_value opt_block_arg + | arg_value tCOMMA tSTAR arg_value opt_block_arg { - result = self.arg_concat(s(:array, val[0]), val[3]); - result = self.arg_blk_pass(result, val[4]); + result = self.arg_concat s(:array, val[0]), val[3] + result = self.arg_blk_pass result, val[4] } - | arg_value ',' args ',' tSTAR arg_value opt_block_arg + | arg_value tCOMMA args tCOMMA tSTAR arg_value opt_block_arg { - result = self.arg_concat(s(:array, val[0], s(:hash, *val[2].values)), val[5]) - result = self.arg_blk_pass(result, val[6]); + result = self.arg_concat s(:array, val[0], s(:hash, *val[2].values)), val[5] + result = self.arg_blk_pass result, val[6] } | assocs opt_block_arg { - result = s(:array, s(:hash, *val[0].values)); - result = self.arg_blk_pass(result, val[1]); + result = s(:array, s(:hash, *val[0].values)) + result = self.arg_blk_pass result, val[1] } - | assocs ',' tSTAR arg_value opt_block_arg + | assocs tCOMMA tSTAR arg_value opt_block_arg { result = s(:array, s(:hash, *val[0].values), val[3]) - result = self.arg_blk_pass(result, val[4]) + result = self.arg_blk_pass result, val[4] } - | arg_value ',' assocs opt_block_arg + | arg_value tCOMMA assocs opt_block_arg { result = s(:array, val[0], s(:hash, *val[2].values)) - result = self.arg_blk_pass(result, val[3]) + result = self.arg_blk_pass result, val[3] } - | arg_value ',' args ',' assocs opt_block_arg + | arg_value tCOMMA args tCOMMA assocs opt_block_arg { - result = s(:array, val[0]).add_all(val[2]).add(s(:hash, *val[4].values)); - result = self.arg_blk_pass(result, val[5]); + result = s(:array, val[0]).add_all(val[2]).add(s(:hash, *val[4].values)) + result = self.arg_blk_pass result, val[5] } - | arg_value ',' assocs ',' tSTAR arg_value opt_block_arg + | arg_value tCOMMA assocs tCOMMA tSTAR arg_value opt_block_arg { - result = self.arg_concat(s(:array, val[0]).add(s(:hash, *val[2].values)), val[5]); - result = self.arg_blk_pass(result, val[6]); + result = self.arg_concat s(:array, val[0]).add(s(:hash, *val[2].values)), val[5] + result = self.arg_blk_pass result, val[6] } - | arg_value ',' args ',' assocs ',' tSTAR arg_value opt_block_arg + | arg_value tCOMMA args tCOMMA assocs tCOMMA tSTAR arg_value opt_block_arg { - result = self.arg_concat(s(:array, val[0]).add_all(val[2]).add(s(:hash, *val[4].values)), val[7]); - result = self.arg_blk_pass(result, val[8]); + result = self.arg_concat s(:array, val[0]).add_all(val[2]).add(s(:hash, *val[4].values)), val[7] + result = self.arg_blk_pass result, val[8] } | tSTAR arg_value opt_block_arg { - result = self.arg_blk_pass(s(:splat, val[1]), val[2]); + result = self.arg_blk_pass s(:splat, val[1]), val[2] } | block_arg @@ -856,8 +855,8 @@ rule } tRPAREN { - warning("don't put space before argument parentheses"); - result = nil; + warning "don't put space before argument parentheses" + result = nil } | tLPAREN_ARG call_args2 { @@ -865,18 +864,18 @@ rule } tRPAREN { - warning("don't put space before argument parentheses"); - result = val[1]; + warning "don't put space before argument parentheses" + result = val[1] } block_arg: tAMPER arg_value { - result = s(:block_pass, val[1]); + result = s(:block_pass, val[1]) } - opt_block_arg: ',' block_arg + opt_block_arg: tCOMMA block_arg { - result = val[1]; + result = val[1] } | none_block_pass @@ -884,18 +883,18 @@ rule { result = s(:array, val[0]) } - | args ',' arg_value + | args tCOMMA arg_value { - result = self.list_append(val[0], val[2]) + result = self.list_append val[0], val[2] } - mrhs: args ',' arg_value + mrhs: args tCOMMA arg_value { result = val[0] << val[2] } - | args ',' tSTAR arg_value + | args tCOMMA tSTAR arg_value { - result = self.arg_concat(val[0], val[3]) + result = self.arg_concat val[0], val[3] } | tSTAR arg_value { @@ -934,8 +933,8 @@ rule } opt_nl tRPAREN { - warning("(...) interpreted as grouped expression"); - result = val[1]; + warning "(...) interpreted as grouped expression" + result = val[1] } | tLPAREN compstmt tRPAREN { @@ -950,7 +949,7 @@ rule { result = s(:colon3, val[1].to_sym) } - | primary_value '[' aref_args tRBRACK + | primary_value "[" aref_args tRBRACK { result = new_aref val } @@ -980,7 +979,7 @@ rule } | kDEFINED opt_nl tLPAREN2 expr tRPAREN { - result = s(:defined, val[3]); + result = s(:defined, val[3]) } | operation brace_block { @@ -1061,7 +1060,7 @@ rule { self.comments.push self.lexer.comments if (self.in_def || self.in_single > 0) then - yyerror("class definition in method body"); + yyerror "class definition in method body" end self.env.extend } @@ -1097,15 +1096,15 @@ rule cpath { self.comments.push self.lexer.comments - yyerror("module definition in method body") if + yyerror "module definition in method body" if self.in_def or self.in_single > 0 - self.env.extend; + self.env.extend } bodystmt kEND { result = new_module val - self.env.unextend; + self.env.unextend } | kDEF fname { @@ -1128,14 +1127,14 @@ rule fname { self.in_single += 1 - self.env.extend; + self.env.extend lexer.lex_state = :expr_end # force for args } f_arglist bodystmt kEND { result = new_defs val - self.env.unextend; + self.env.unextend self.in_single -= 1 } | kBREAK @@ -1161,24 +1160,24 @@ rule } then: term - | ":" + | tCOLON | kTHEN | term kTHEN do: term - | ":" + | tCOLON | kDO_COND if_tail: opt_else | kELSIF expr_value then compstmt if_tail { - result = s(:if, val[1], val[3], val[4]); + result = s(:if, val[1], val[3], val[4]) } opt_else: none | kELSE compstmt { - result = val[1]; + result = val[1] } block_var: lhs @@ -1198,7 +1197,7 @@ rule } | tPIPE block_var tPIPE { - result = val[1]; + result = val[1] } do_block: kDO_BLOCK @@ -1312,13 +1311,13 @@ rule } when_args: args - | args ',' tSTAR arg_value + | args tCOMMA tSTAR arg_value { - result = self.list_append(val[0], s(:when, val[3], nil)) + result = self.list_append val[0], s(:when, val[3], nil) } | tSTAR arg_value { - result = s(:array, s(:when, val[1], nil)); + result = s(:array, s(:when, val[1], nil)) } cases: opt_else | case_body @@ -1340,21 +1339,21 @@ rule exc_list: arg_value { - result = s(:array, val[0]); + result = s(:array, val[0]) } | mrhs | none exc_var: tASSOC lhs { - result = val[1]; + result = val[1] } | none opt_ensure: kENSURE compstmt { if (val[1] != nil) then - result = val[1]; + result = val[1] else result = s(:nil) end @@ -1368,13 +1367,13 @@ rule strings: string { val[0] = s(:dstr, val[0].value) if val[0][0] == :evstr - result = val[0]; + result = val[0] } string: string1 | string string1 { - result = self.literal_concat(val[0], val[1]); + result = self.literal_concat val[0], val[1] } string1: tSTRING_BEG string_contents tSTRING_END @@ -1396,32 +1395,32 @@ rule result = new_regexp val } - words: tWORDS_BEG ' ' tSTRING_END + words: tWORDS_BEG tSPACE tSTRING_END { - result = s(:array); + result = s(:array) } | tWORDS_BEG word_list tSTRING_END { - result = val[1]; + result = val[1] } word_list: none { result = s(:array) } - | word_list word ' ' + | word_list word tSPACE { - word = val[1][0] == :evstr ? s(:dstr, '', val[1]) : val[1] + word = val[1][0] == :evstr ? s(:dstr, "", val[1]) : val[1] result = val[0] << word } word: string_content | word string_content { - result = self.literal_concat(val[0], val[1]); + result = self.literal_concat val[0], val[1] } - awords: tAWORDS_BEG ' ' tSTRING_END + awords: tAWORDS_BEG tSPACE tSTRING_END { result = s(:array) } @@ -1434,7 +1433,7 @@ rule { result = s(:array) } - | qword_list tSTRING_CONTENT ' ' + | qword_list tSTRING_CONTENT tSPACE { result = val[0] << s(:str, val[1]) } @@ -1463,18 +1462,18 @@ xstring_contents: none } | tSTRING_DVAR { - result = lexer.lex_strterm; + result = lexer.lex_strterm lexer.lex_strterm = nil lexer.lex_state = :expr_beg } string_dvar { lexer.lex_strterm = val[1] - result = s(:evstr, val[2]); + result = s(:evstr, val[2]) } | tSTRING_DBEG { - result = lexer.lex_strterm; + result = lexer.lex_strterm lexer.lex_strterm = nil lexer.lex_state = :expr_beg lexer.cond.push false @@ -1524,7 +1523,7 @@ xstring_contents: none lexer.lex_state = :expr_end result = val[1] - yyerror("empty symbol literal") if + yyerror "empty symbol literal" if result.nil? or result.empty? case result[0] @@ -1533,7 +1532,7 @@ xstring_contents: none when :str then result = s(:lit, result.last.intern) else - result = s(:dsym, '', result) + result = s(:dsym, "", result) end } @@ -1553,21 +1552,21 @@ xstring_contents: none | tGVAR | tCONSTANT | tCVAR - | kNIL { result = :nil } - | kSELF { result = :self } - | kTRUE { result = :true } - | kFALSE { result = :false } - | k__FILE__ { result = :"__FILE__" } - | k__LINE__ { result = :"__LINE__" } + | kNIL { result = :nil } + | kSELF { result = :self } + | kTRUE { result = :true } + | kFALSE { result = :false } + | k__FILE__ { result = :__FILE__ } + | k__LINE__ { result = :__LINE__ } var_ref: variable { - result = self.gettable(val[0]) + result = self.gettable val[0] } var_lhs: variable { - result = self.assignable(val[0]); + result = self.assignable val[0] } backref: tNTH_REF { result = s(:nth_ref, val[0]) } @@ -1583,33 +1582,33 @@ xstring_contents: none } expr_value term { - result = val[2]; + result = val[2] } | error term { - yyerrok; - result = nil; + yyerrok + result = nil } f_arglist: tLPAREN2 f_args opt_nl tRPAREN { - result = val[1]; + result = val[1] lexer.lex_state = :expr_beg } | f_args term { - result = val[0]; + result = val[0] } - f_args: f_arg ',' f_optarg ',' f_rest_arg opt_f_block_arg + f_args: f_arg tCOMMA f_optarg tCOMMA f_rest_arg opt_f_block_arg { result = args val[0], val[2], val[4], val[5] } - | f_arg ',' f_optarg opt_f_block_arg + | f_arg tCOMMA f_optarg opt_f_block_arg { result = args val[0], val[2], nil, val[3] } - | f_arg ',' f_rest_arg opt_f_block_arg + | f_arg tCOMMA f_rest_arg opt_f_block_arg { result = args val[0], nil, val[2], val[3] } @@ -1617,7 +1616,7 @@ xstring_contents: none { result = args val[0], nil, nil, val[1] } - | f_optarg ',' f_rest_arg opt_f_block_arg + | f_optarg tCOMMA f_rest_arg opt_f_block_arg { result = args nil, val[0], val[2], val[3] } @@ -1644,18 +1643,18 @@ xstring_contents: none } | tIVAR { - yyerror("formal argument cannot be an instance variable"); + yyerror "formal argument cannot be an instance variable" } | tCVAR { - yyerror("formal argument cannot be a class variable"); + yyerror "formal argument cannot be a class variable" } | tIDENTIFIER { identifier = val[0].to_sym self.env[identifier] = :lvar - result = val[0]; + result = val[0] } f_arg: f_norm_arg @@ -1663,15 +1662,15 @@ xstring_contents: none result = s(:args) result << val[0].to_sym } - | f_arg ',' f_norm_arg + | f_arg tCOMMA f_norm_arg { val[0] << val[2].to_sym result = val[0] } - f_opt: tIDENTIFIER '=' arg_value + f_opt: tIDENTIFIER tEQL arg_value { - result = self.assignable(val[0], val[2]); + result = self.assignable val[0], val[2] # TODO: detect duplicate names } @@ -1679,9 +1678,9 @@ xstring_contents: none { result = s(:block, val[0]) } - | f_optarg ',' f_opt + | f_optarg tCOMMA f_opt { - result = self.append_to_block(val[0], val[2]); + result = self.append_to_block val[0], val[2] } restarg_mark: tSTAR2 | tSTAR @@ -1690,7 +1689,7 @@ xstring_contents: none { # TODO: differs from parse.y - needs tests name = val[1].to_sym - self.assignable(name) + self.assignable name result = :"*#{name}" } | restarg_mark @@ -1710,13 +1709,13 @@ xstring_contents: none result = s(:block_arg, identifier.to_sym) } - opt_f_block_arg: ',' f_block_arg + opt_f_block_arg: tCOMMA f_block_arg { - result = val[1]; + result = val[1] } | { - result = nil; + result = nil } singleton: var_ref @@ -1727,7 +1726,7 @@ xstring_contents: none expr opt_nl tRPAREN { result = val[2] - yyerror("Can't define single method for literals.") if + yyerror "Can't define single method for literals." if result[0] == :lit } @@ -1737,19 +1736,19 @@ xstring_contents: none } | assocs trailer # [!nil] { - result = val[0]; + result = val[0] } | args trailer { size = val[0].size if (size % 2 != 1) then # != 1 because of leading :array - yyerror("Odd number (#{size}) list for Hash. #{val[0].inspect}"); + yyerror "Odd number (#{size}) list for Hash. #{val[0].inspect}" end - result = val[0]; + result = val[0] } assocs: assoc - | assocs ',' assoc + | assocs tCOMMA assoc { list = val[0].dup more = val[2][1..-1] @@ -1767,14 +1766,14 @@ xstring_contents: none operation3: tIDENTIFIER | tFID | op dot_or_colon: tDOT | tCOLON2 opt_terms: | terms - opt_nl: | "\n" - trailer: | "\n" | ',' + opt_nl: | tNL + trailer: | tNL | tCOMMA - term: ';' { yyerrok } - | "\n" + term: tSEMI { yyerrok } + | tNL terms: term - | terms ';' { yyerrok } + | terms tSEMI { yyerrok } none: { result = nil } @@ -1784,7 +1783,7 @@ end ---- inner -require 'ruby_lexer' +require "ruby_lexer" # Local Variables: ** # racc-token-length-max:14 ** diff --git a/test/test_ruby_lexer.rb b/test/test_ruby_lexer.rb index 451a7cd0..71d6daaf 100755 --- a/test/test_ruby_lexer.rb +++ b/test/test_ruby_lexer.rb @@ -1,7 +1,7 @@ #!/usr/local/bin/ruby -require 'test/unit' -require 'ruby_lexer' +require "test/unit" +require "ruby_lexer" class TestRubyLexer < Test::Unit::TestCase def deny cond, msg = nil @@ -22,21 +22,21 @@ def test_advance end def test_read_escape - util_escape "\\", '\\' - util_escape "\n", 'n' - util_escape "\t", 't' - util_escape "\r", 'r' - util_escape "\f", 'f' - util_escape "\13", 'v' - util_escape "\0", '0' - util_escape "\07", 'a' - util_escape "\007", 'a' - util_escape "\033", 'e' - util_escape "\377", '377' - util_escape "\377", 'xff' - util_escape "\010", 'b' - util_escape " ", 's' - util_escape "q", 'q' # plain vanilla escape + util_escape "\\", "\\" + util_escape "\n", "n" + util_escape "\t", "t" + util_escape "\r", "r" + util_escape "\f", "f" + util_escape "\13", "v" + util_escape "\0", "0" + util_escape "\07", "a" + util_escape "\007", "a" + util_escape "\033", "e" + util_escape "\377", "377" + util_escape "\377", "xff" + util_escape "\010", "b" + util_escape " ", "s" + util_escape "q", "q" # plain vanilla escape end def test_read_escape_c @@ -128,9 +128,9 @@ def test_yylex_assoc def test_yylex_back_ref util_lex_token("[$&, $`, $', $+]", :tLBRACK, "[", - :tBACK_REF, :"&", ",", ",", - :tBACK_REF, :"`", ",", ",", - :tBACK_REF, :"'", ",", ",", + :tBACK_REF, :"&", :tCOMMA, ",", + :tBACK_REF, :"`", :tCOMMA, ",", + :tBACK_REF, :"'", :tCOMMA, ",", :tBACK_REF, :"+", :tRBRACK, "]") end @@ -216,13 +216,13 @@ def test_yylex_colon3 end def test_yylex_comma - util_lex_token ",", ",", "," # FIX + util_lex_token ",", :tCOMMA, "," end def test_yylex_comment util_lex_token("1 # one\n# two\n2", :tINTEGER, 1, - "\n", nil, + :tNL, nil, :tINTEGER, 2) assert_equal "# one\n# two\n", @lex.comments end @@ -235,18 +235,18 @@ def test_yylex_comment_begin def test_yylex_comment_begin_bad util_bad_token("=begin\nblah\nblah\n") - assert_equal '', @lex.comments + assert_equal "", @lex.comments end def test_yylex_comment_begin_not_comment util_lex_token("beginfoo = 5\np x \\\n=beginfoo", :tIDENTIFIER, "beginfoo", - '=', '=', + :tEQL, "=", :tINTEGER, 5, - "\n", nil, + :tNL, nil, :tIDENTIFIER, "p", :tIDENTIFIER, "x", - '=', '=', + :tEQL, "=", :tIDENTIFIER, "beginfoo") end @@ -267,7 +267,7 @@ def test_yylex_constant def test_yylex_constant_semi util_lex_token("ArgumentError;", :tCONSTANT, "ArgumentError", - ";", ";") + :tSEMI, ";") end def test_yylex_cvar @@ -340,7 +340,7 @@ def test_yylex_do_cond end def test_yylex_dollar - util_lex_token('$', '$', '$') # FIX: wtf is this?!? + util_lex_token("$", "$", "$") # FIX: wtf is this?!? end def test_yylex_dot # HINT message sends @@ -356,7 +356,7 @@ def test_yylex_dot3 end def test_yylex_equals - util_lex_token "=", '=', "=" # FIX: this sucks + util_lex_token "=", :tEQL, "=" # FIX: this sucks end def test_yylex_equals2 @@ -390,8 +390,8 @@ def test_yylex_float_bad_trailing_underscore def test_yylex_float_call util_lex_token("1.0.to_s", :tFLOAT, 1.0, - :tDOT, '.', - :tIDENTIFIER, 'to_s') + :tDOT, ".", + :tIDENTIFIER, "to_s") end def test_yylex_float_dot_E @@ -479,11 +479,11 @@ def test_yylex_global_backref end def test_yylex_global_dash_nothing - util_lex_token('$- ', :tGVAR, "$-") + util_lex_token("$- ", :tGVAR, "$-") end def test_yylex_global_dash_something - util_lex_token('$-x', :tGVAR, "$-x") + util_lex_token("$-x", :tGVAR, "$-x") end def test_yylex_global_number @@ -497,23 +497,23 @@ def test_yylex_global_number_big end def test_yylex_global_other - util_lex_token('[$~, $*, $$, $?, $!, $@, $/, $\, $;, $,, $., $=, $:, $<, $>, $"]', + util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]", :tLBRACK, "[", - :tGVAR, "$~", ",", ",", - :tGVAR, "$*", ",", ",", - :tGVAR, "$$", ",", ",", - :tGVAR, "$?", ",", ",", - :tGVAR, "$!", ",", ",", - :tGVAR, "$@", ",", ",", - :tGVAR, "$/", ",", ",", - :tGVAR, "$\\", ",", ",", - :tGVAR, "$;", ",", ",", - :tGVAR, "$,", ",", ",", - :tGVAR, "$.", ",", ",", - :tGVAR, "$=", ",", ",", - :tGVAR, "$:", ",", ",", - :tGVAR, "$<", ",", ",", - :tGVAR, "$>", ",", ",", + :tGVAR, "$~", :tCOMMA, ",", + :tGVAR, "$*", :tCOMMA, ",", + :tGVAR, "$$", :tCOMMA, ",", + :tGVAR, "$\?", :tCOMMA, ",", + :tGVAR, "$!", :tCOMMA, ",", + :tGVAR, "$@", :tCOMMA, ",", + :tGVAR, "$/", :tCOMMA, ",", + :tGVAR, "$\\", :tCOMMA, ",", + :tGVAR, "$;", :tCOMMA, ",", + :tGVAR, "$,", :tCOMMA, ",", + :tGVAR, "$.", :tCOMMA, ",", + :tGVAR, "$=", :tCOMMA, ",", + :tGVAR, "$:", :tCOMMA, ",", + :tGVAR, "$<", :tCOMMA, ",", + :tGVAR, "$>", :tCOMMA, ",", :tGVAR, "$\"", :tRBRACK, "]") end @@ -529,7 +529,7 @@ def test_yylex_global_wierd end def test_yylex_global_zero - util_lex_token('$0', :tGVAR, "$0") + util_lex_token("$0", :tGVAR, "$0") end def test_yylex_gt @@ -542,51 +542,51 @@ def test_yylex_gt def test_yylex_heredoc_backtick util_lex_token("a = <<`EOF`\n blah blah\nEOF\n", :tIDENTIFIER, "a", - "=", "=", + :tEQL, "=", :tXSTRING_BEG, "`", :tSTRING_CONTENT, " blah blah\n", :tSTRING_END, "EOF", - "\n", nil) + :tNL, nil) end def test_yylex_heredoc_double util_lex_token("a = <<\"EOF\"\n blah blah\nEOF\n", :tIDENTIFIER, "a", - "=", "=", + :tEQL, "=", :tSTRING_BEG, "\"", :tSTRING_CONTENT, " blah blah\n", :tSTRING_END, "EOF", - "\n", nil) + :tNL, nil) end def test_yylex_heredoc_double_dash util_lex_token("a = <<-\"EOF\"\n blah blah\n EOF\n", :tIDENTIFIER, "a", - "=", "=", + :tEQL, "=", :tSTRING_BEG, "\"", :tSTRING_CONTENT, " blah blah\n", :tSTRING_END, "EOF", - "\n", nil) + :tNL, nil) end def test_yylex_heredoc_double_eos util_bad_token("a = <<\"EOF\"\nblah", :tIDENTIFIER, "a", - "=", "=", + :tEQL, "=", :tSTRING_BEG, "\"") end def test_yylex_heredoc_double_eos_nl util_bad_token("a = <<\"EOF\"\nblah\n", :tIDENTIFIER, "a", - "=", "=", + :tEQL, "=", :tSTRING_BEG, "\"") end def test_yylex_heredoc_double_interp util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n", :tIDENTIFIER, "a", - "=", "=", + :tEQL, "=", :tSTRING_BEG, "\"", :tSTRING_CONTENT, "#x a ", :tSTRING_DVAR, "\#@", @@ -596,84 +596,84 @@ def test_yylex_heredoc_double_interp :tSTRING_DBEG, "\#{", :tSTRING_CONTENT, "3} \n", # HUH? :tSTRING_END, "EOF", - "\n", nil) + :tNL, nil) end def test_yylex_heredoc_none util_lex_token("a = <