Permalink
Browse files

+ Renamed ruby_parser.y to ruby19_parser.y

+ Added RubyParser, subclassing Ruby18Parser but warning on instantiation.
+ Added ruby_parser.rb that pulls everything together in proper order.
+ RubyLexer now takes a version specifier.
- Fixed lexing of ?c for ruby 1.8 and 1.9.
+ Moved everything from RubyParser to RubyParserStuff and included module in both.
- Duplicate the input so that heredoc processing doesn't morph original. (banister)
+ Added more 18 vs 19 lexing tests for ?c.
+ Refactored tests infrastructure and added both 1.8 and 1.9 test branches.

[git-p4: depot-paths = "//src/ruby_parser/dev/": change = 6709]
  • Loading branch information...
zenspider committed Oct 17, 2011
1 parent 04803f5 commit 705272813396e1f6cb5ce27add900e98840b76a4
View
@@ -5,9 +5,11 @@ README.txt
Rakefile
bin/ruby_parse
lib/gauntlet_rubyparser.rb
-lib/ruby_lexer.rb
-lib/ruby_parser.y
+lib/ruby18_parser.rb
lib/ruby18_parser.y
+lib/ruby19_parser.rb
+lib/ruby19_parser.y
+lib/ruby_lexer.rb
lib/ruby_parser.rb
lib/ruby_parser_extras.rb
test/test_ruby_lexer.rb
View
@@ -16,7 +16,10 @@ Hoe.spec 'ruby_parser' do
dependency 'sexp_processor', '~> 3.0'
- self.perforce_ignore << "lib/ruby_parser.rb" if plugin? :perforce
+ if plugin? :perforce then
+ self.perforce_ignore << "lib/ruby18_parser.rb"
+ self.perforce_ignore << "lib/ruby19_parser.rb"
+ end
self.racc_flags << " -g" if plugin?(:racc) && ENV["DEBUG"]
end
View
@@ -1,6 +1,6 @@
# -*- racc -*-
-class RubyParser
+class Ruby18Parser
token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
kTHEN kELSIF kELSE kCASE kWHEN kWHILE kUNTIL kFOR kBREAK kNEXT
@@ -1,6 +1,6 @@
# -*- racc -*-
-class RubyParser
+class Ruby19Parser
token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
kTHEN kELSIF kELSE kCASE kWHEN kWHILE kUNTIL kFOR kBREAK kNEXT
View
@@ -6,6 +6,12 @@ class RubyLexer
ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
+ ##
+ # What version of ruby to parse. 18 and 19 are the only valid values
+ # currently supported.
+
+ attr_accessor :version
+
# Additional context surrounding tokens that both the lexer and
# grammar use.
attr_reader :lex_state
@@ -217,7 +223,8 @@ def heredoc_identifier # 51 lines
end
end
- def initialize
+ def initialize v = 18
+ self.version = v
self.cond = RubyParser::StackState.new(:cond)
self.cmdarg = RubyParser::StackState.new(:cmdarg)
self.nest = 0
@@ -1044,8 +1051,14 @@ def yylex # 826 lines
src.getch
end
self.lex_state = :expr_end
- self.yacc_value = c
- return :tSTRING
+
+ if version == 18 then
+ self.yacc_value = c[0].ord & 0xff
+ return :tINTEGER
+ else
+ self.yacc_value = c
+ return :tSTRING
+ end
elsif src.check(/\&/) then
if src.scan(/\&\&\=/) then
self.yacc_value = "&&"
View
@@ -123,7 +123,7 @@ def scan re
# end
end
-class RubyParser < Racc::Parser
+module RubyParserStuff
VERSION = '2.3.1' unless constants.include? "VERSION" # SIGH
attr_accessor :lexer, :in_def, :in_single, :file
@@ -334,7 +334,9 @@ def gettable(id)
def initialize(options = {})
super()
- self.lexer = RubyLexer.new
+
+ v = self.class.name[/1[89]/]
+ self.lexer = RubyLexer.new v && v.to_i
self.lexer.parser = self
@env = Environment.new
@comments = []
@@ -745,7 +747,7 @@ def process(str, file = "(string)")
raise "bad val: #{str.inspect}" unless String === str
self.file = file
- self.lexer.src = str
+ self.lexer.src = str.dup
@yydebug = ENV.has_key? 'DEBUG'
@@ -811,10 +813,9 @@ def warning s
# do nothing for now
end
- alias :old_yyerror :yyerror
def yyerror msg
# for now do nothing with the msg
- old_yyerror
+ super
end
class Keyword
@@ -1000,6 +1001,22 @@ def push val
end
end
+class Ruby19Parser < Racc::Parser
+ include RubyParserStuff
+end
+
+class Ruby18Parser < Racc::Parser
+ include RubyParserStuff
+end
+
+class RubyParser < Ruby18Parser
+ def initialize
+ super
+ warn "WA\RNING: Deprecated: RubyParser. Use Ruby18Parser or Ruby19Parser"
+ warn " from #{caller.first}"
+ end
+end
+
############################################################
# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
View
@@ -1,15 +1,17 @@
#!/usr/local/bin/ruby
require 'rubygems'
+gem "minitest"
+
require 'minitest/autorun'
require 'ruby_lexer'
-require 'ruby_parser'
+require 'ruby18_parser'
class TestRubyLexer < MiniTest::Unit::TestCase
alias :deny :refute
def setup
- p = RubyParser.new
+ p = Ruby18Parser.new
@lex = p.lexer
@lex.src = "blah blah"
@lex.lex_state = :expr_beg
@@ -805,11 +807,27 @@ def test_yylex_integer_dec_d_bad_underscores
util_bad_token "0d42__24"
end
- def test_yylex_question_eh_a
+ def test_yylex_question_eh_a__18
+ @lex = RubyLexer.new 18
+
+ util_lex_token "?a", :tINTEGER, 97
+ end
+
+ def test_yylex_question_eh_a__19
+ @lex = RubyLexer.new 19
+
util_lex_token '?a', :tSTRING, "a"
end
- def test_yylex_question_eh_escape_M_escape_C
+ def test_yylex_question_eh_escape_M_escape_C__18
+ @lex = RubyLexer.new 18
+
+ util_lex_token '?\M-\C-a', :tINTEGER, 129
+ end
+
+ def test_yylex_question_eh_escape_M_escape_C__19
+ @lex = RubyLexer.new 19
+
util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
end
@@ -1089,7 +1107,15 @@ def test_yylex_plus_unary_number
:tINTEGER, 42)
end
- def test_yylex_question
+ def test_yylex_question__18
+ @lex = RubyLexer.new 18
+
+ util_lex_token "?*", :tINTEGER, 42
+ end
+
+ def test_yylex_question__19
+ @lex = RubyLexer.new 19
+
util_lex_token "?*", :tSTRING, "*"
end
@@ -1106,7 +1132,26 @@ def test_yylex_question_ws
util_lex_token "?\f", :tEH, "?"
end
- def test_yylex_question_ws_backslashed
+ def test_yylex_question_ws_backslashed__18
+ @lex = RubyLexer.new 18
+
+ @lex.lex_state = :expr_beg
+ util_lex_token "?\\ ", :tINTEGER, 32
+ @lex.lex_state = :expr_beg
+ util_lex_token "?\\n", :tINTEGER, 10
+ @lex.lex_state = :expr_beg
+ util_lex_token "?\\t", :tINTEGER, 9
+ @lex.lex_state = :expr_beg
+ util_lex_token "?\\v", :tINTEGER, 11
+ @lex.lex_state = :expr_beg
+ util_lex_token "?\\r", :tINTEGER, 13
+ @lex.lex_state = :expr_beg
+ util_lex_token "?\\f", :tINTEGER, 12
+ end
+
+ def test_yylex_question_ws_backslashed__19
+ @lex = RubyLexer.new 19
+
@lex.lex_state = :expr_beg
util_lex_token "?\\ ", :tSTRING, " "
@lex.lex_state = :expr_beg
View
@@ -3,20 +3,29 @@
ENV['VERBOSE'] = "1"
require 'rubygems'
+gem "minitest"
require 'minitest/autorun'
require 'ruby_parser'
$: << File.expand_path('~/Work/p4/zss/src/ParseTree/dev/test')
require 'pt_testcase'
-class RubyParser
+class Ruby18Parser # FIX
+ def process input
+ parse input
+ end
+end
+
+class Ruby19Parser
def process input
parse input
end
end
class RubyParserTestCase < ParseTreeTestCase
+ attr_accessor :result, :processor
+
def self.previous key
"Ruby"
end
@@ -29,16 +38,6 @@ def self.generate_test klass, node, data, input_name, output_name
super
end
-end
-
-class TestRubyParser < RubyParserTestCase
- attr_accessor :result, :processor
-
- def setup
- super
-
- self.processor = RubyParser.new
- end
def assert_parse rb, pt
self.result = processor.parse rb
@@ -49,6 +48,30 @@ def assert_parse_line rb, pt, line
assert_parse rb, pt
assert_equal line, result.line, "call should have line number"
end
+end
+
+class TestRuby18Parser < RubyParserTestCase
+ def setup
+ super
+
+ self.processor = Ruby18Parser.new
+ end
+end
+
+class TestRuby19Parser < RubyParserTestCase
+ def setup
+ super
+
+ self.processor = Ruby19Parser.new
+ end
+end
+
+class XTestRubyParser # < RubyParserTestCase
+ def setup
+ super
+
+ self.processor = RubyParser.new
+ end
def test_attrasgn_array_lhs
rb = '[1, 2, 3, 4][from .. to] = ["a", "b", "c"]'
@@ -1,7 +1,10 @@
require 'rubygems'
+gem "minitest"
require 'minitest/autorun'
require 'ruby_parser_extras'
+require 'minitest/unit'
+
class TestStackState < MiniTest::Unit::TestCase
attr_reader :s

0 comments on commit 7052728

Please sign in to comment.