@@ -2,4 +2,7 @@ *.so a.out Makefile - +ext/rmmseg +doc +*.gem +!ext/rmmseg/*.{cpp,h} .gitignore @@ -20,7 +20,7 @@ module RMMSeg end def token_stream(field, text) - t = PunctuationFilter.new(Tokenizer.new(text)) + t = Tokenizer.new(text) if @brk @brk.call(t) else @@ -61,56 +61,5 @@ module RMMSeg @algor = Algorithm.new(@text) end end - - # PunctuationFilter filter out the stand alone Chinese - # punctuation tokens. - class PunctuationFilter < ::Ferret::Analysis::TokenStream - # The punctuation dictionary. - class Dictionary - include Singleton - - DIC_FILE = File.join(File.dirname(__FILE__), - "..", - "..", - "data", - "punctuation.dic") - def initialize - @dic = Hash.new - File.open(DIC_FILE, "r") do |f| - f.each_line { |line| - @dic[line.chomp.freeze] = nil - } - end - end - - def include?(str) - @dic.has_key?(str) - end - end - - def initialize(stream) - @stream = stream - end - - # Get next token, skip stand alone Chinese punctuations. - def next - token = @stream.next - dic = Dictionary.instance - - until token.nil? || !(dic.include? token.text) - token = @stream.next - end - - token - end - - def text - @stream.text - end - - def text=(str) - @stream.text = str - end - end end end lib/rmmseg/ferret.rb @@ -19,3 +19,4 @@ spec = Gem::Specification.new do |s| '--line-numbers' s.extra_rdoc_files = ['README'] end + rmmseg-cpp.gemspec data/punctuation.dic 1ad1d3ba8a3477b9145dd70fb5aef15cef851e6d pluskid pluskid@gmail.com http://github.com/pluskid/rmmseg-cpp/commit/9036639e199ac43be5af3d79ac8b5c4e9c3ccbd4 9036639e199ac43be5af3d79ac8b5c4e9c3ccbd4 2008-05-22T04:17:54-07:00 2008-05-22T04:17:34-07:00 Removed punctuation filter. It is duplicated with Ferret Stopword filter. a5943cafc4f5e82a3a9efff9cb6ca39495a87a1e pluskid pluskid@gmail.com