+ Renamed ruby_parser.y to ruby19_parser.y

+ Added RubyParser, subclassing Ruby18Parser but warning on instantiation.
+ Added ruby_parser.rb that pulls everything together in proper order.
+ RubyLexer now takes a version specifier.
- Fixed lexing of ?c for ruby 1.8 and 1.9.
+ Moved everything from RubyParser to RubyParserStuff and included module in both.
- Duplicate the input so that heredoc processing doesn't morph original. (banister)
+ Added more 18 vs 19 lexing tests for ?c.
+ Refactored tests infrastructure and added both 1.8 and 1.9 test branches.

[git-p4: depot-paths = "//src/ruby_parser/dev/": change = 6709]

Manifest.txt

@@ -5,9 +5,11 @@ README.txt
 Rakefile
 bin/ruby_parse
 lib/gauntlet_rubyparser.rb
-lib/ruby_lexer.rb
-lib/ruby_parser.y
+lib/ruby18_parser.rb
 lib/ruby18_parser.y
+lib/ruby19_parser.rb
+lib/ruby19_parser.y
+lib/ruby_lexer.rb
 lib/ruby_parser.rb
 lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb

Rakefile

@@ -16,7 +16,10 @@ Hoe.spec 'ruby_parser' do
 
   dependency 'sexp_processor', '~> 3.0'
 
-  self.perforce_ignore << "lib/ruby_parser.rb" if plugin? :perforce
+  if plugin? :perforce then
+    self.perforce_ignore << "lib/ruby18_parser.rb"
+    self.perforce_ignore << "lib/ruby19_parser.rb"
+  end
 
   self.racc_flags << " -g" if plugin?(:racc) && ENV["DEBUG"]
 end

lib/ruby18_parser.y

@@ -1,6 +1,6 @@
 # -*- racc -*-
 
-class RubyParser
+class Ruby18Parser
 
 token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       kTHEN kELSIF kELSE kCASE kWHEN kWHILE kUNTIL kFOR kBREAK kNEXT

lib/ruby_parser.y → lib/ruby19_parser.y

@@ -1,6 +1,6 @@
 # -*- racc -*-
 
-class RubyParser
+class Ruby19Parser
 
 token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       kTHEN kELSIF kELSE kCASE kWHEN kWHILE kUNTIL kFOR kBREAK kNEXT

lib/ruby_lexer.rb

@@ -6,6 +6,12 @@ class RubyLexer
 
   ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
 
+  ##
+  # What version of ruby to parse. 18 and 19 are the only valid values
+  # currently supported.
+
+  attr_accessor :version
+
   # Additional context surrounding tokens that both the lexer and
   # grammar use.
   attr_reader :lex_state
@@ -217,7 +223,8 @@ def heredoc_identifier # 51 lines
     end
   end
 
-  def initialize
+  def initialize v = 18
+    self.version = v
     self.cond = RubyParser::StackState.new(:cond)
     self.cmdarg = RubyParser::StackState.new(:cmdarg)
     self.nest = 0
@@ -1044,8 +1051,14 @@ def yylex # 826 lines
                 src.getch
               end
           self.lex_state = :expr_end
-          self.yacc_value = c
-          return :tSTRING
+
+          if version == 18 then
+            self.yacc_value = c[0].ord & 0xff
+            return :tINTEGER
+          else
+            self.yacc_value = c
+            return :tSTRING
+          end
         elsif src.check(/\&/) then
           if src.scan(/\&\&\=/) then
             self.yacc_value = "&&"

lib/ruby_parser_extras.rb

@@ -123,7 +123,7 @@ def scan re
   # end
 end
 
-class RubyParser < Racc::Parser
+module RubyParserStuff
   VERSION = '2.3.1' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
@@ -334,7 +334,9 @@ def gettable(id)
 
   def initialize(options = {})
     super()
-    self.lexer = RubyLexer.new
+
+    v = self.class.name[/1[89]/]
+    self.lexer = RubyLexer.new v && v.to_i
     self.lexer.parser = self
     @env = Environment.new
     @comments = []
@@ -745,7 +747,7 @@ def process(str, file = "(string)")
     raise "bad val: #{str.inspect}" unless String === str
 
     self.file = file
-    self.lexer.src = str
+    self.lexer.src = str.dup
 
     @yydebug = ENV.has_key? 'DEBUG'
 
@@ -811,10 +813,9 @@ def warning s
     # do nothing for now
   end
 
-  alias :old_yyerror :yyerror
   def yyerror msg
     # for now do nothing with the msg
-    old_yyerror
+    super
   end
 
   class Keyword
@@ -1000,6 +1001,22 @@ def push val
   end
 end
 
+class Ruby19Parser < Racc::Parser
+  include RubyParserStuff
+end
+
+class Ruby18Parser < Racc::Parser
+  include RubyParserStuff
+end
+
+class RubyParser < Ruby18Parser
+  def initialize
+    super
+    warn "WA\RNING: Deprecated: RubyParser. Use Ruby18Parser or Ruby19Parser"
+    warn "  from #{caller.first}"
+  end
+end
+
 ############################################################
 # HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
 

test/test_ruby_lexer.rb

@@ -1,15 +1,17 @@
 #!/usr/local/bin/ruby
 
 require 'rubygems'
+gem "minitest"
+
 require 'minitest/autorun'
 require 'ruby_lexer'
-require 'ruby_parser'
+require 'ruby18_parser'
 
 class TestRubyLexer < MiniTest::Unit::TestCase
   alias :deny :refute
 
   def setup
-    p = RubyParser.new
+    p = Ruby18Parser.new
     @lex = p.lexer
     @lex.src = "blah blah"
     @lex.lex_state = :expr_beg
@@ -805,11 +807,27 @@ def test_yylex_integer_dec_d_bad_underscores
     util_bad_token "0d42__24"
   end
 
-  def test_yylex_question_eh_a
+  def test_yylex_question_eh_a__18
+    @lex = RubyLexer.new 18
+
+    util_lex_token "?a", :tINTEGER, 97
+  end
+
+  def test_yylex_question_eh_a__19
+    @lex = RubyLexer.new 19
+
     util_lex_token '?a', :tSTRING, "a"
   end
 
-  def test_yylex_question_eh_escape_M_escape_C
+  def test_yylex_question_eh_escape_M_escape_C__18
+    @lex = RubyLexer.new 18
+
+    util_lex_token '?\M-\C-a', :tINTEGER, 129
+  end
+
+  def test_yylex_question_eh_escape_M_escape_C__19
+    @lex = RubyLexer.new 19
+
     util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
   end
 
@@ -1089,7 +1107,15 @@ def test_yylex_plus_unary_number
                    :tINTEGER, 42)
   end
 
-  def test_yylex_question
+  def test_yylex_question__18
+    @lex = RubyLexer.new 18
+
+    util_lex_token "?*", :tINTEGER, 42
+  end
+
+  def test_yylex_question__19
+    @lex = RubyLexer.new 19
+
     util_lex_token "?*", :tSTRING, "*"
   end
 
@@ -1106,7 +1132,26 @@ def test_yylex_question_ws
     util_lex_token "?\f", :tEH, "?"
   end
 
-  def test_yylex_question_ws_backslashed
+  def test_yylex_question_ws_backslashed__18
+    @lex = RubyLexer.new 18
+
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\ ", :tINTEGER, 32
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\n", :tINTEGER, 10
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\t", :tINTEGER, 9
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\v", :tINTEGER, 11
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\r", :tINTEGER, 13
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\f", :tINTEGER, 12
+  end
+
+  def test_yylex_question_ws_backslashed__19
+    @lex = RubyLexer.new 19
+
     @lex.lex_state = :expr_beg
     util_lex_token "?\\ ", :tSTRING, " "
     @lex.lex_state = :expr_beg

test/test_ruby_parser.rb

@@ -3,20 +3,29 @@
 ENV['VERBOSE'] = "1"
 
 require 'rubygems'
+gem "minitest"
 require 'minitest/autorun'
 require 'ruby_parser'
 
 $: << File.expand_path('~/Work/p4/zss/src/ParseTree/dev/test')
 
 require 'pt_testcase'
 
-class RubyParser
+class Ruby18Parser # FIX
+  def process input
+    parse input
+  end
+end
+
+class Ruby19Parser
   def process input
     parse input
   end
 end
 
 class RubyParserTestCase < ParseTreeTestCase
+  attr_accessor :result, :processor
+
   def self.previous key
     "Ruby"
   end
@@ -29,16 +38,6 @@ def self.generate_test klass, node, data, input_name, output_name
 
     super
   end
-end
-
-class TestRubyParser < RubyParserTestCase
-  attr_accessor :result, :processor
-
-  def setup
-    super
-
-    self.processor = RubyParser.new
-  end
 
   def assert_parse rb, pt
     self.result = processor.parse rb
@@ -49,6 +48,30 @@ def assert_parse_line rb, pt, line
     assert_parse rb, pt
     assert_equal line, result.line,   "call should have line number"
   end
+end
+
+class TestRuby18Parser < RubyParserTestCase
+  def setup
+    super
+
+    self.processor = Ruby18Parser.new
+  end
+end
+
+class TestRuby19Parser < RubyParserTestCase
+  def setup
+    super
+
+    self.processor = Ruby19Parser.new
+  end
+end
+
+class XTestRubyParser # < RubyParserTestCase
+  def setup
+    super
+
+    self.processor = RubyParser.new
+  end
 
   def test_attrasgn_array_lhs
     rb = '[1, 2, 3, 4][from .. to] = ["a", "b", "c"]'

test/test_ruby_parser_extras.rb

@@ -1,7 +1,10 @@
 require 'rubygems'
+gem "minitest"
 require 'minitest/autorun'
 require 'ruby_parser_extras'
 
+require 'minitest/unit'
+
 class TestStackState < MiniTest::Unit::TestCase
   attr_reader :s