@@ -49,7 +49,7 @@ Please not that Windows is not supported at this time. $ git clone git://github.com/aurelian/ruby-stemmer.git $ cd ruby-stemmer $ rake -T #<== see what we've got - $ rake ext #<== builds the extension do'h + $ rake compile #<== builds the extension do'h $ rake test == NOT A BUG README.rdoc @@ -1,20 +1,10 @@ #include "ruby.h" #include <libstemmer.h> -#define GetStemmer(obj, sb_data) {\ - Data_Get_Struct(obj, struct sb_stemmer_data, sb_data);\ -} - VALUE rb_mLingua; VALUE rb_cStemmer; VALUE rb_eStemmerError; -struct sb_stemmer_data { - struct sb_stemmer * stemmer; - const char * lang; - const char * enc; -}; - /* * Document-method: new * call-seq: Lingua::Stemmer.new @@ -26,48 +16,27 @@ struct sb_stemmer_data { * s = Lingua::Stemmer.new :language => 'fr' */ static VALUE -rb_stemmer_init(int argc, VALUE *argv, VALUE self) { - VALUE roptions, rlang, renc; - +rb_stemmer_init(VALUE self, VALUE rlang, VALUE renc) { struct sb_stemmer * stemmer; - struct sb_stemmer_data *sb_data; - rb_scan_args(argc, argv, "01", &roptions); - - if(argc > 0) { - Check_Type(roptions, T_HASH); - if((rlang = rb_hash_aref(roptions, ID2SYM(rb_intern("language")))) != Qnil) { - Check_Type(rlang, T_STRING); - } else { - rlang = rb_str_new2("en"); - } - if((renc = rb_hash_aref(roptions, ID2SYM(rb_intern("encoding")))) != Qnil) { - Check_Type(renc, T_STRING); - } else { - renc = rb_str_new2("UTF_8"); - } - } else { - rlang = rb_str_new2("en"); - renc = rb_str_new2("UTF_8"); - } - + Data_Get_Struct(self, struct sb_stemmer, stemmer); + + // In case someone sends() this method, free up the old one + if(stemmer) sb_stemmer_delete(stemmer); + stemmer = sb_stemmer_new( RSTRING_PTR(rlang), RSTRING_PTR(renc) ); - if (stemmer == 0) { - if (renc == 0 ) { - rb_raise(rb_eStemmerError, "Language %s not available for stemming", RSTRING_PTR(rlang)); - exit(1); + if (!stemmer) { + if (!RTEST(renc)) { + rb_raise(rb_eStemmerError, + "Language %s not available for stemming", RSTRING_PTR(rlang)); } else { - rb_raise(rb_eStemmerError, "Language %s not available for stemming in encoding %s", + rb_raise(rb_eStemmerError, + "Language %s not available for stemming in encoding %s", RSTRING_PTR(rlang), RSTRING_PTR(renc)); - exit(1); } } - sb_data = ALLOC(struct sb_stemmer_data); - DATA_PTR(self) = sb_data; - sb_data->stemmer= stemmer; - sb_data->lang = RSTRING_PTR(rlang); - sb_data->enc = RSTRING_PTR(renc); + DATA_PTR(self) = stemmer; return self; } @@ -84,52 +53,23 @@ rb_stemmer_init(int argc, VALUE *argv, VALUE self) { */ static VALUE rb_stemmer_stem(VALUE self, VALUE word) { - struct sb_stemmer_data * sb_data; - const sb_symbol * stemmed; - VALUE s_word = rb_String(word); - GetStemmer(self, sb_data); - stemmed = sb_stemmer_stem(sb_data->stemmer, (sb_symbol *)RSTRING_PTR(s_word), RSTRING_LEN(s_word)); - return rb_str_new2((char *)stemmed); -} + struct sb_stemmer * stemmer; -/* - * Document-method: language - * call-seq: language - * - * Gets the language for this stemmer - * - * require 'lingua/stemmer' - * s = Lingua::Stemmer.new(:language => "fr") - * s.language #=> "fr" - */ -static VALUE -rb_stemmer_language(VALUE self) { - struct sb_stemmer_data * sb_data; - GetStemmer(self, sb_data); - return rb_str_new2(sb_data->lang); -} + Data_Get_Struct(self, struct sb_stemmer, stemmer); + if(!stemmer) rb_raise(rb_eRuntimeError, "Stemmer not initialize"); -/* - * Document-method: encoding - * call-seq: encoding - * - * Gets the encoding for this stemmer - * - * require 'lingua/stemmer' - * s = Lingua::Stemmer.new(:language => "UTF_8") - * s.encoding #=> "UTF_8" - */ -static VALUE -rb_stemmer_encoding(VALUE self) { - struct sb_stemmer_data * sb_data; - GetStemmer(self, sb_data); - return rb_str_new2(sb_data->enc); + VALUE s_word = rb_String(word); + const sb_symbol * stemmed = sb_stemmer_stem(stemmer, + (sb_symbol *)RSTRING_PTR(s_word), + RSTRING_LEN(s_word) + ); + return rb_str_new2((char *)stemmed); } static void -sb_stemmer_free(struct sb_stemmer_data * sb_data) +sb_stemmer_free(struct sb_stemmer * stemmer) { - sb_stemmer_delete(sb_data->stemmer); + if(stemmer) sb_stemmer_delete(stemmer); } static VALUE @@ -146,9 +86,7 @@ void Init_stemmer_native() { rb_cStemmer = rb_define_class_under(rb_mLingua, "Stemmer", rb_cObject); rb_define_alloc_func(rb_cStemmer, sb_stemmer_alloc); rb_eStemmerError = rb_define_class_under(rb_mLingua, "StemmerError", rb_eException); - rb_define_method(rb_cStemmer, "initialize", rb_stemmer_init, -1); + rb_define_private_method(rb_cStemmer, "native_init", rb_stemmer_init, 2); rb_define_method(rb_cStemmer, "stem", rb_stemmer_stem, 1); - rb_define_method(rb_cStemmer, "language", rb_stemmer_language, 0); - rb_define_method(rb_cStemmer, "encoding", rb_stemmer_encoding, 0); } ext/lingua/stemmer.c @@ -5,24 +5,42 @@ else end module Lingua - def self.stemmer(o, options={}) - _stemmer= Stemmer.new({:language => "en", :encoding => "UTF_8"}.merge(options)) - words= o.kind_of?(Array)? o.map{|e|e.to_s} : [o.to_s] - results = [] unless block_given? - words.each do | word | - result = _stemmer.stem(word) + stemmer = Stemmer.new(options) + + words = Array(o).map { |e| e.to_s } + + results = [] + words.each do |word| + result = stemmer.stem(word) if block_given? yield result else results << result end end - return (results.length == 1)? results[0] : results unless block_given? - _stemmer + + return stemmer if block_given? + results.length == 1 ? results[0] : results end class Stemmer VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "VERSION"))).strip! + + attr_reader :language + attr_reader :encoding + + # Creates a new Stemmer, pass <tt>:language</tt> and <tt>:encoding</tt> + # as arguments to change encoding or language, otherwise english with UTF_8 + # will be used + # + # require 'lingua/stemmer' + # s = Lingua::Stemmer.new :language => 'fr' + # + def initialize options = {} + @language = (options[:language] || 'en').to_s + @encoding = (options[:encoding] || 'UTF_8').to_s + native_init @language, @encoding + end end end lib/lingua/stemmer.rb @@ -38,4 +38,11 @@ class TestStemmer < Test::Unit::TestCase assert_kind_of Array, results end + def test_stemmer_subclass + assert_raises(RuntimeError) do + Class.new(Lingua::Stemmer) { + def native_init a, b; end + }.new.stem('cow') + end + end end test/lingua/test_stemmer.rb 38ccbd5339ffdb918631508029e1ee4f9a0fdec3 Aaron Patterson aaron.patterson@gmail.com http://github.com/aurelian/ruby-stemmer/commit/d939c73bec7292d7b9d7e92f5100338fa1979fff d939c73bec7292d7b9d7e92f5100338fa1979fff 2009-11-02T11:49:45-08:00 2009-11-02T11:49:45-08:00 cleaning up C and Ruby codes 8c7b4a38641ab353ed5e44563ddc79d79009cdff Aaron Patterson aaron.patterson@gmail.com