Rakefile~ filtering-with-tokenise.diff @@ -12,7 +12,8 @@ Hoe.new("pci4r", Pci4R::VERSION) do |p| p.url = "http://github.com/alexvollmer/pci4r/tree/master" p.extra_deps = [ ['activerecord', '>=2.0.0'], - ['sqlite3-ruby', '>=1.2.1'] + ['sqlite3-ruby', '>=1.2.1'], + ['stemmer' , '>=1.0.1'] ] end Rakefile @@ -1,4 +1,5 @@ require "set" +require "stemmer" ## # This module provides a number of +Classifier+ classes that are instantiated, @@ -516,4 +517,62 @@ module Filtering [sum, 1.0].min end end -end \ No newline at end of file +end + +class String + + @@n_gram_length = 1 + + # These are copied uncritically from Lucas Carlson's Classifier library. + @@stop_words = %w(a again all along are also an and as at but by came can cant couldnt did didn didnt do doesnt dont ever first from have her here him how i if in into is isnt it itll just last least like most my new no not now of on or should sinc so some th than this that the their then those to told too true try until url us were when whether while with within yes you youll) + + def self.n_gram_length=(newval) + @@n_gram_length = newval + end + + def self.stop_words=(newval) + @@stop_words = newval + end + def self.stop_words + @@stop_words + end + def self.add_stop_words(adds) + @@stop_words += adds.map {|w| w.downcase} + @@stop_words.uniq! + end + def self.remove_stop_words(removes) + @@stop_words -= removes + end + def tokenise + # Doing some regex massaging. + # Replace single or double dashes w/a single space. Replace tildes w/spaces + text = gsub(/(-{1,2}|~)/, ' ').downcase + + # Eat punctuation + text.gsub!(/[^\w\s]/, '') + + # Change digits to placeholder + text.gsub!(/\d/, '#') + + words = text.split.select {|w| w.length > 1 and not(@@stop_words.include?(w))} + + tokens = [] + 0.upto(words.length) do |i| + @@n_gram_length.downto(1) do |k| + chunk = words[i, k] + if chunk.length == k then + # TODO?: keep from crossing obvious sentence boundaries (e.g., test for periods in elements other than chunk[-1]) + tokens << chunk.map do |w| + # puts(w) + w.stem + end.join('_') + end + end + end + + return tokens # + words.map {|w| w.stem} + + end + +end + lib/filtering.rb @@ -4,6 +4,40 @@ require File.join(File.dirname(__FILE__), "..", "lib", "filtering") require "tempfile" describe "Filtering" do + describe "tokenise" do + it "should ignore stop words" do + @words = "I told him to ignore most of these".tokenise + @words.should_not be_member("told".stem) + @words.should_not be_member("him".stem) + @words.should be_member("ignore".stem) + @words.should be_member("these".stem) + end + + it "should honor added stop words" do + String.add_stop_words(%w(ignore all these)) + @words = "I told him to ignore most of these".tokenise + @words.should_not be_member("told".stem) + @words.should_not be_member("him".stem) + @words.should_not be_member("ignore".stem) + @words.should_not be_member("these".stem) + end + + it "should honor removed stop words" do + String.remove_stop_words(%w(dont not cant couldnt)) + @words = "I don't want you trying to do things you can't".tokenise + @words.should be_member("dont".stem) + @words.should be_member("cant".stem) + end + + it "should tokenise properly" do + str = "The fisherman fished a fish out of the fishing holes" + expected = %w(fisherman fish fish out fish hole) + str.tokenise.should == expected + String.n_gram_length = 2 + expected = %w(fisherman_fish fisherman fish_fish fish fish_out fish out_fish out fish_hole fish hole) + str.tokenise.should == expected + end + end describe "get_words" do before(:each) do @@ -130,6 +164,11 @@ describe "Filtering" do @classifier.minimums[:bad] = 0.8 @classifier.classify("quick money").should == :good end + + it "should handle long documents w/out barfing" do + @classifier.classify("make penis fast " * 100000).should == :bad + end + end describe "ActiveRecord persistence" do @@ -161,5 +200,6 @@ describe "Filtering" do @classifier.feature_count("dog", :good).should == 1.0 @classifier.feature_count("dog", :bad).should == 0.0 end + end -end \ No newline at end of file +end spec/filtering_spec.rb 6fa6608acc023e95be1579ed72b63bdd612b9e96 Roy Pardee roy@kif.(none) http://github.com/alexvollmer/pci4r/commit/4d88b00958defd05827217b27bfb4daf6b0a7084 4d88b00958defd05827217b27bfb4daf6b0a7084 2009-04-12T12:40:49-07:00 2009-04-12T12:40:49-07:00 Added reference to Stemmer lib to hoe...spec? a2e9891e9f0def52dadcc97fff62c865cf8c84dd Roy Pardee roy@kif.(none)