Add ripper's .rb files to our stdlib.

lib/ruby/1.9/ripper.rb

@@ -0,0 +1,4 @@
+require 'ripper/core'
+require 'ripper/lexer'
+require 'ripper/filter'
+require 'ripper/sexp'

lib/ruby/1.9/ripper/core.rb

@@ -0,0 +1,70 @@
+#
+# $Id$
+#
+# Copyright (c) 2003-2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper.so'
+
+class Ripper
+
+  # Parses Ruby program read from _src_.
+  # _src_ must be a String or a IO or a object which has #gets method.
+  def Ripper.parse(src, filename = '(ripper)', lineno = 1)
+    new(src, filename, lineno).parse
+  end
+
+  # This array contains name of parser events.
+  PARSER_EVENTS = PARSER_EVENT_TABLE.keys
+
+  # This array contains name of scanner events.
+  SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
+
+  # This array contains name of all ripper events.
+  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+  private
+
+  #
+  # Parser Events
+  #
+
+  PARSER_EVENT_TABLE.each do |id, arity|
+    module_eval(<<-End, __FILE__, __LINE__ + 1)
+      def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') })
+        #{arity == 0 ? 'nil' : 'a'}
+      end
+    End
+  end
+
+  # This method is called when weak warning is produced by the parser.
+  # _fmt_ and _args_ is printf style.
+  def warn(fmt, *args)
+  end
+
+  # This method is called when strong warning is produced by the parser.
+  # _fmt_ and _args_ is printf style.
+  def warning(fmt, *args)
+  end
+
+  # This method is called when the parser found syntax error.
+  def compile_error(msg)
+  end
+
+  #
+  # Scanner Events
+  #
+
+  SCANNER_EVENTS.each do |id|
+    module_eval(<<-End, __FILE__, __LINE__ + 1)
+      def on_#{id}(token)
+        token
+      end
+    End
+  end
+
+end

lib/ruby/1.9/ripper/filter.rb

@@ -0,0 +1,70 @@
+#
+# $Id$
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/lexer'
+
+class Ripper
+
+  # This class handles only scanner events,
+  # and they are dispatched in the `right' order (same with input).
+  class Filter
+
+    def initialize(src, filename = '-', lineno = 1)
+      @__lexer = Lexer.new(src, filename, lineno)
+      @__line = nil
+      @__col = nil
+    end
+
+    # The file name of the input.
+    def filename
+      @__lexer.filename
+    end
+
+    # The line number of the current token.
+    # This value starts from 1.
+    # This method is valid only in event handlers.
+    def lineno
+      @__line
+    end
+
+    # The column number of the current token.
+    # This value starts from 0.
+    # This method is valid only in event handlers.
+    def column
+      @__col
+    end
+
+    # Starts parsing.  _init_ is a data accumulator.
+    # It is passed to the next event handler (as of Enumerable#inject).
+    def parse(init = nil)
+      data = init
+      @__lexer.lex.each do |pos, event, tok|
+        @__line, @__col = *pos
+        data = if respond_to?(event, true)
+               then __send__(event, tok, data)
+               else on_default(event, tok, data)
+               end
+      end
+      data
+    end
+
+    private
+
+    # This method is called when some event handler have not defined.
+    # _event_ is :on_XXX, _token_ is scanned token, _data_ is a data
+    # accumulator.  The return value of this method is passed to the
+    # next event handler (as of Enumerable#inject).
+    def on_default(event, token, data)
+      data
+    end
+
+  end
+
+end

lib/ruby/1.9/ripper/lexer.rb

@@ -0,0 +1,179 @@
+#
+# $Id$
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/core'
+
+class Ripper
+
+  # Tokenizes Ruby program and returns an Array of String.
+  def Ripper.tokenize(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).tokenize
+  end
+
+  # Tokenizes Ruby program and returns an Array of Array,
+  # which is formatted like [[lineno, column], type, token].
+  #
+  #   require 'ripper'
+  #   require 'pp'
+  #
+  #   p Ripper.lex("def m(a) nil end")
+  #     #=> [[[1,  0], :on_kw,     "def"],
+  #          [[1,  3], :on_sp,     " "  ],
+  #          [[1,  4], :on_ident,  "m"  ],
+  #          [[1,  5], :on_lparen, "("  ],
+  #          [[1,  6], :on_ident,  "a"  ],
+  #          [[1,  7], :on_rparen, ")"  ],
+  #          [[1,  8], :on_sp,     " "  ],
+  #          [[1,  9], :on_kw,     "nil"],
+  #          [[1, 12], :on_sp,     " "  ],
+  #          [[1, 13], :on_kw,     "end"]]
+  #
+  def Ripper.lex(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).lex
+  end
+
+  class Lexer < ::Ripper   #:nodoc: internal use only
+    def tokenize
+      lex().map {|pos, event, tok| tok }
+    end
+
+    def lex
+      parse().sort_by {|pos, event, tok| pos }
+    end
+
+    def parse
+      @buf = []
+      super
+      @buf
+    end
+
+    private
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
+        def on_#{event}(tok)
+          @buf.push [[lineno(), column()], :on_#{event}, tok]
+        end
+      End
+    end
+  end
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and return a string which was matched to +pattern+.
+  # +pattern+ should be described as Regexp.
+  #
+  #   require 'ripper'
+  #
+  #   p Ripper.slice('def m(a) nil end', 'ident')                   #=> "m"
+  #   p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+')  #=> "m(a)"
+  #   p Ripper.slice("<<EOS\nstring\nEOS",
+  #                  'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
+  #       #=> "string\n"
+  #
+  def Ripper.slice(src, pattern, n = 0)
+    if m = token_match(src, pattern)
+    then m.string(n)
+    else nil
+    end
+  end
+
+  def Ripper.token_match(src, pattern)   #:nodoc:
+    TokenPattern.compile(pattern).match(src)
+  end
+
+  class TokenPattern   #:nodoc:
+
+    class Error < ::StandardError; end
+    class CompileError < Error; end
+    class MatchError < Error; end
+
+    class << self
+      alias compile new
+    end
+
+    def initialize(pattern)
+      @source = pattern
+      @re = compile(pattern)
+    end
+
+    def match(str)
+      match_list(::Ripper.lex(str))
+    end
+
+    def match_list(tokens)
+      if m = @re.match(map_tokens(tokens))
+      then MatchData.new(tokens, m)
+      else nil
+      end
+    end
+
+    private
+
+    def compile(pattern)
+      if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
+        raise CompileError, "invalid char in pattern: #{m[0].inspect}"
+      end
+      buf = ''
+      pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
+        case tok
+        when /\w/
+          buf.concat map_token(tok)
+        when '$('
+          buf.concat '('
+        when '('
+          buf.concat '(?:'
+        when /[?*\[\])\.]/
+          buf.concat tok
+        else
+          raise 'must not happen'
+        end
+      end
+      Regexp.compile(buf)
+    rescue RegexpError => err
+      raise CompileError, err.message
+    end
+
+    def map_tokens(tokens)
+      tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
+    end
+
+    MAP = {}
+    seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+    SCANNER_EVENT_TABLE.each do |ev, |
+      raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
+      MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
+    end
+
+    def map_token(tok)
+      MAP[tok]  or raise CompileError, "unknown token: #{tok}"
+    end
+
+    class MatchData
+      def initialize(tokens, match)
+        @tokens = tokens
+        @match = match
+      end
+
+      def string(n = 0)
+        return nil unless @match
+        match(n).join
+      end
+
+      private
+
+      def match(n = 0)
+        return [] unless @match
+        @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
+      end
+    end
+
+  end
+
+end