-
Notifications
You must be signed in to change notification settings - Fork 152
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
f52c9ca
commit 9a6546a
Showing
7 changed files
with
431 additions
and
0 deletions.
There are no files selected for viewing
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
require 'ripper/core' | ||
require 'ripper/lexer' | ||
require 'ripper/filter' | ||
require 'ripper/sexp' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
# | ||
# $Id: core.rb 25189 2009-10-02 12:04:37Z akr $ | ||
# | ||
# Copyright (c) 2003-2005 Minero Aoki | ||
# | ||
# This program is free software. | ||
# You can distribute and/or modify this program under the Ruby License. | ||
# For details of Ruby License, see ruby/COPYING. | ||
# | ||
|
||
require File.dirname(__FILE__) + '/../../ext/ripper.so' | ||
|
||
class Ripper | ||
|
||
# Parses Ruby program read from _src_. | ||
# _src_ must be a String or a IO or a object which has #gets method. | ||
def Ripper.parse(src, filename = '(ripper)', lineno = 1) | ||
new(src, filename, lineno).parse | ||
end | ||
|
||
# This array contains name of parser events. | ||
PARSER_EVENTS = PARSER_EVENT_TABLE.keys | ||
|
||
# This array contains name of scanner events. | ||
SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys | ||
|
||
# This array contains name of all ripper events. | ||
EVENTS = PARSER_EVENTS + SCANNER_EVENTS | ||
|
||
private | ||
|
||
# | ||
# Parser Events | ||
# | ||
|
||
PARSER_EVENT_TABLE.each do |id, arity| | ||
module_eval(<<-End, __FILE__, __LINE__ + 1) | ||
def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') }) | ||
#{arity == 0 ? 'nil' : 'a'} | ||
end | ||
End | ||
end | ||
|
||
# This method is called when weak warning is produced by the parser. | ||
# _fmt_ and _args_ is printf style. | ||
def warn(fmt, *args) | ||
end | ||
|
||
# This method is called when strong warning is produced by the parser. | ||
# _fmt_ and _args_ is printf style. | ||
def warning(fmt, *args) | ||
end | ||
|
||
# This method is called when the parser found syntax error. | ||
def compile_error(msg) | ||
end | ||
|
||
# | ||
# Scanner Events | ||
# | ||
|
||
SCANNER_EVENTS.each do |id| | ||
module_eval(<<-End, __FILE__, __LINE__ + 1) | ||
def on_#{id}(token) | ||
token | ||
end | ||
End | ||
end | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
# | ||
# $Id: filter.rb 25189 2009-10-02 12:04:37Z akr $ | ||
# | ||
# Copyright (c) 2004,2005 Minero Aoki | ||
# | ||
# This program is free software. | ||
# You can distribute and/or modify this program under the Ruby License. | ||
# For details of Ruby License, see ruby/COPYING. | ||
# | ||
|
||
require 'ripper/lexer' | ||
|
||
class Ripper | ||
|
||
# This class handles only scanner events, | ||
# and they are dispatched in the `right' order (same with input). | ||
class Filter | ||
|
||
def initialize(src, filename = '-', lineno = 1) | ||
@__lexer = Lexer.new(src, filename, lineno) | ||
@__line = nil | ||
@__col = nil | ||
end | ||
|
||
# The file name of the input. | ||
def filename | ||
@__lexer.filename | ||
end | ||
|
||
# The line number of the current token. | ||
# This value starts from 1. | ||
# This method is valid only in event handlers. | ||
def lineno | ||
@__line | ||
end | ||
|
||
# The column number of the current token. | ||
# This value starts from 0. | ||
# This method is valid only in event handlers. | ||
def column | ||
@__col | ||
end | ||
|
||
# Starts parsing. _init_ is a data accumulator. | ||
# It is passed to the next event handler (as of Enumerable#inject). | ||
def parse(init = nil) | ||
data = init | ||
@__lexer.lex.each do |pos, event, tok| | ||
@__line, @__col = *pos | ||
data = if respond_to?(event, true) | ||
then __send__(event, tok, data) | ||
else on_default(event, tok, data) | ||
end | ||
end | ||
data | ||
end | ||
|
||
private | ||
|
||
# This method is called when some event handler have not defined. | ||
# _event_ is :on_XXX, _token_ is scanned token, _data_ is a data | ||
# accumulator. The return value of this method is passed to the | ||
# next event handler (as of Enumerable#inject). | ||
def on_default(event, token, data) | ||
data | ||
end | ||
|
||
end | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,179 @@ | ||
# | ||
# $Id: lexer.rb 25189 2009-10-02 12:04:37Z akr $ | ||
# | ||
# Copyright (c) 2004,2005 Minero Aoki | ||
# | ||
# This program is free software. | ||
# You can distribute and/or modify this program under the Ruby License. | ||
# For details of Ruby License, see ruby/COPYING. | ||
# | ||
|
||
require 'ripper/core' | ||
|
||
class Ripper | ||
|
||
# Tokenizes Ruby program and returns an Array of String. | ||
def Ripper.tokenize(src, filename = '-', lineno = 1) | ||
Lexer.new(src, filename, lineno).tokenize | ||
end | ||
|
||
# Tokenizes Ruby program and returns an Array of Array, | ||
# which is formatted like [[lineno, column], type, token]. | ||
# | ||
# require 'ripper' | ||
# require 'pp' | ||
# | ||
# p Ripper.lex("def m(a) nil end") | ||
# #=> [[[1, 0], :on_kw, "def"], | ||
# [[1, 3], :on_sp, " " ], | ||
# [[1, 4], :on_ident, "m" ], | ||
# [[1, 5], :on_lparen, "(" ], | ||
# [[1, 6], :on_ident, "a" ], | ||
# [[1, 7], :on_rparen, ")" ], | ||
# [[1, 8], :on_sp, " " ], | ||
# [[1, 9], :on_kw, "nil"], | ||
# [[1, 12], :on_sp, " " ], | ||
# [[1, 13], :on_kw, "end"]] | ||
# | ||
def Ripper.lex(src, filename = '-', lineno = 1) | ||
Lexer.new(src, filename, lineno).lex | ||
end | ||
|
||
class Lexer < ::Ripper #:nodoc: internal use only | ||
def tokenize | ||
lex().map {|pos, event, tok| tok } | ||
end | ||
|
||
def lex | ||
parse().sort_by {|pos, event, tok| pos } | ||
end | ||
|
||
def parse | ||
@buf = [] | ||
super | ||
@buf | ||
end | ||
|
||
private | ||
|
||
SCANNER_EVENTS.each do |event| | ||
module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1) | ||
def on_#{event}(tok) | ||
@buf.push [[lineno(), column()], :on_#{event}, tok] | ||
end | ||
End | ||
end | ||
end | ||
|
||
# [EXPERIMENTAL] | ||
# Parses +src+ and return a string which was matched to +pattern+. | ||
# +pattern+ should be described as Regexp. | ||
# | ||
# require 'ripper' | ||
# | ||
# p Ripper.slice('def m(a) nil end', 'ident') #=> "m" | ||
# p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)" | ||
# p Ripper.slice("<<EOS\nstring\nEOS", | ||
# 'heredoc_beg nl $(tstring_content*) heredoc_end', 1) | ||
# #=> "string\n" | ||
# | ||
def Ripper.slice(src, pattern, n = 0) | ||
if m = token_match(src, pattern) | ||
then m.string(n) | ||
else nil | ||
end | ||
end | ||
|
||
def Ripper.token_match(src, pattern) #:nodoc: | ||
TokenPattern.compile(pattern).match(src) | ||
end | ||
|
||
class TokenPattern #:nodoc: | ||
|
||
class Error < ::StandardError; end | ||
class CompileError < Error; end | ||
class MatchError < Error; end | ||
|
||
class << self | ||
alias compile new | ||
end | ||
|
||
def initialize(pattern) | ||
@source = pattern | ||
@re = compile(pattern) | ||
end | ||
|
||
def match(str) | ||
match_list(::Ripper.lex(str)) | ||
end | ||
|
||
def match_list(tokens) | ||
if m = @re.match(map_tokens(tokens)) | ||
then MatchData.new(tokens, m) | ||
else nil | ||
end | ||
end | ||
|
||
private | ||
|
||
def compile(pattern) | ||
if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern) | ||
raise CompileError, "invalid char in pattern: #{m[0].inspect}" | ||
end | ||
buf = '' | ||
pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok| | ||
case tok | ||
when /\w/ | ||
buf.concat map_token(tok) | ||
when '$(' | ||
buf.concat '(' | ||
when '(' | ||
buf.concat '(?:' | ||
when /[?*\[\])\.]/ | ||
buf.concat tok | ||
else | ||
raise 'must not happen' | ||
end | ||
end | ||
Regexp.compile(buf) | ||
rescue RegexpError => err | ||
raise CompileError, err.message | ||
end | ||
|
||
def map_tokens(tokens) | ||
tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join | ||
end | ||
|
||
MAP = {} | ||
seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a | ||
SCANNER_EVENT_TABLE.each do |ev, | | ||
raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty? | ||
MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift | ||
end | ||
|
||
def map_token(tok) | ||
MAP[tok] or raise CompileError, "unknown token: #{tok}" | ||
end | ||
|
||
class MatchData | ||
def initialize(tokens, match) | ||
@tokens = tokens | ||
@match = match | ||
end | ||
|
||
def string(n = 0) | ||
return nil unless @match | ||
match(n).join | ||
end | ||
|
||
private | ||
|
||
def match(n = 0) | ||
return [] unless @match | ||
@tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str } | ||
end | ||
end | ||
|
||
end | ||
|
||
end |
Oops, something went wrong.