@@ -1,13 +1,8 @@ -# the filter currently checks for squeezed words that start out at 3 or more characters. -# example - will filter 'ass' and 'aasssss' but not 'as' - still working on a better way to avoid assess TODO - reserved word list. -# keep in mind, the squeezed version of words (shitter shiter) when adding terms +# the filter currently checks for words that are 3 or more characters. --- -as: "*ss" ass: "*ss" asses: "*ss*s" -ashole: "*ssh*l*" asshole: "*ssh*l*" -asholes: "*ssh*l*s" assholes: "*ssh*l*s" bastard: b*st*rd beastial: b**st**l @@ -24,7 +19,6 @@ bitching: b*tch*ng blowjob: bl*wj*b blowjobs: bl*wj*bs bullshit: b*llsh*t -bulshit: b*llsh*t clit: cl*t cock: c*ck cocks: c*cks @@ -38,7 +32,6 @@ cummer: c*mm*r cumming: c*mm*ng cums: c*ms cumshot: c*msh*t -cunilingus: c*n*l*ng*s cunillingus: c*n*ll*ng*s cunnilingus: c*nn*l*ng*s cunt: c*nt @@ -55,6 +48,7 @@ cyberfucking: cyb*rf*ck*ng damn: d*mn dildo: d*ld* dildos: d*ld*s +dick: d*ck dink: d*nk dinks: d*nks ejaculate: "*j*c*l*t*" @@ -203,4 +197,4 @@ slut: sl*t sluts: sl*ts smut: sm*t spunk: sp*nk -twat: tw*t +twat: tw*t \ No newline at end of file config/dictionary.yml @@ -39,16 +39,43 @@ module ProfanityFilter class << self def clean(text, replace_method = '') return text if text.blank? - text.split(/(\W)/).collect{|word| replace_method == 'dictionary' ? clean_word_dictionary(word) : clean_word_basic(word)}.join - end - - def clean_word_dictionary(word) - dictionary.include?(word.downcase.squeeze) && word.size > 2 ? dictionary[word.downcase.squeeze] : word - end - - def clean_word_basic(word) - dictionary.include?(word.downcase.squeeze) && word.size > 2 ? replacement_text : word + @replace_method = replace_method + text.split(/(\s)/).collect{ |word| + clean_word(word) + }.join end + + def clean_word(word) + return word unless(word.strip.size > 2) + + if word.index(/[\W]/) + word = word.split(/(\W)/).collect{ |subword| + clean_word(subword) + }.join + concat = word.gsub(/\W/, '') + word = concat if is_banned? concat + end + + is_banned?(word) ? replacement(word) : word + end + + def replacement(word) + case @replace_method + when 'dictionary' + dictionary[word.downcase] || word + when 'vowels' + word.gsub(/[aeiou]/i, '*') + when 'hollow' + word[1..word.size-2] = '*' * (word.size-2) if word.size > 2 + word + else + replacement_text + end + end + + def is_banned?(word = '') + dictionary.include?(word.downcase) + end end end end lib/profanity_filter.rb @@ -12,11 +12,23 @@ module DictionaryPostHelper end end +module VowelsPostHelper + class Post < ActiveRecord::Base + profanity_filter :title, :body, :method => 'vowels' + end +end + +module HollowPostHelper + class Post < ActiveRecord::Base + profanity_filter :title, :body, :method => 'hollow' + end +end + class BasicProfanityFilterTest < Test::Unit::TestCase include BasicPostHelper def profane_post(opts={}) - Post.new({:title => 'A Fucking Title', :body => "This is some shitty post by a fucking user"}.merge(opts)) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) end def test_it_should_filter_specified_fields @@ -25,9 +37,9 @@ class BasicProfanityFilterTest < Test::Unit::TestCase assert_equal 'A @#$% Title', p.title assert_equal 'A @#$% Title', p.title_clean assert_equal 'A Fucking Title', p.title_original - assert_equal 'This is some @#$% post by a @#$% user', p.body - assert_equal 'This is some @#$% post by a @#$% user', p.body_clean - assert_equal 'This is some shitty post by a fucking user', p.body_original + assert_equal 'This is some @#$% @#$%-post by a @#$% user', p.body + assert_equal 'This is some @#$% @#$%-post by a @#$% user', p.body_clean + assert_equal 'This is some f-u-c-k-i-n-g shitty-post by a fucking user', p.body_original end def test_it_should_handle_nil_fields_bug_9 @@ -49,7 +61,7 @@ class DictionaryProfanityFilterTest < Test::Unit::TestCase include DictionaryPostHelper def profane_post(opts={}) - Post.new({:title => 'A Fucking Title', :body => "This is some shitty post by a fucking user"}.merge(opts)) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) end def test_it_should_filter_specified_fields @@ -58,9 +70,75 @@ class DictionaryProfanityFilterTest < Test::Unit::TestCase assert_equal 'A f*ck*ng Title', p.title assert_equal 'A f*ck*ng Title', p.title_clean assert_equal 'A Fucking Title', p.title_original - assert_equal 'This is some sh*tty post by a f*ck*ng user', p.body - assert_equal 'This is some sh*tty post by a f*ck*ng user', p.body_clean - assert_equal 'This is some shitty post by a fucking user', p.body_original + assert_equal 'This is some f*ck*ng sh*tty-post by a f*ck*ng user', p.body + assert_equal 'This is some f*ck*ng sh*tty-post by a f*ck*ng user', p.body_clean + assert_equal 'This is some f-u-c-k-i-n-g shitty-post by a fucking user', p.body_original + end + + def test_it_should_handle_nil_fields_bug_9 + p = Post.new({:title => nil, :body => nil}) + p.save + assert_equal nil, p.title + assert_equal nil, p.body + end + + def test_it_should_handle_blank_fields_bug_9 + p = Post.new({:title => "", :body => ""}) + p.save + assert_equal "", p.title + assert_equal "", p.body + end +end + +class VowelsProfanityFilterTest < Test::Unit::TestCase + include VowelsPostHelper + + def profane_post(opts={}) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) + end + + def test_it_should_filter_specified_fields + p = profane_post + p.save + assert_equal 'A F*ck*ng Title', p.title + assert_equal 'A F*ck*ng Title', p.title_clean + assert_equal 'A Fucking Title', p.title_original + assert_equal 'This is some f*ck*ng sh*tty-post by a f*ck*ng user', p.body + assert_equal 'This is some f*ck*ng sh*tty-post by a f*ck*ng user', p.body_clean + assert_equal 'This is some f-u-c-k-i-n-g shitty-post by a fucking user', p.body_original + end + + def test_it_should_handle_nil_fields_bug_9 + p = Post.new({:title => nil, :body => nil}) + p.save + assert_equal nil, p.title + assert_equal nil, p.body + end + + def test_it_should_handle_blank_fields_bug_9 + p = Post.new({:title => "", :body => ""}) + p.save + assert_equal "", p.title + assert_equal "", p.body + end +end + +class HollowProfanityFilterTest < Test::Unit::TestCase + include HollowPostHelper + + def profane_post(opts={}) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) + end + + def test_it_should_filter_specified_fields + p = profane_post + p.save + assert_equal 'A F*****g Title', p.title + assert_equal 'A F*****g Title', p.title_clean + assert_equal 'A Fucking Title', p.title_original + assert_equal 'This is some f*****g s****y-post by a f*****g user', p.body + assert_equal 'This is some f*****g s****y-post by a f*****g user', p.body_clean + assert_equal 'This is some f-u-c-k-i-n-g shitty-post by a fucking user', p.body_original end def test_it_should_handle_nil_fields_bug_9 test/benign_filter_test.rb @@ -12,18 +12,31 @@ module DictionaryPostHelper end end +module VowelsPostHelper + class Post < ActiveRecord::Base + profanity_filter! :title, :body, :method => 'vowels' + end +end + +module HollowPostHelper + class Post < ActiveRecord::Base + profanity_filter! :title, :body, :method => 'hollow' + end +end + + class BasicProfanityFilterTest < Test::Unit::TestCase include BasicPostHelper def profane_post(opts={}) - Post.new({:title => 'A Fucking Title', :body => "This is some shitty post by a fucking user"}.merge(opts)) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) end def test_it_should_filter_specified_fields p = profane_post p.save assert_equal 'A @#$% Title', p.title - assert_equal 'This is some @#$% post by a @#$% user', p.body + assert_equal 'This is some @#$% @#$%-post by a @#$% user', p.body end def test_it_should_handle_nil_fields_bug_9 @@ -45,14 +58,72 @@ class DictionaryProfanityFilterTest < Test::Unit::TestCase include DictionaryPostHelper def profane_post(opts={}) - Post.new({:title => 'A Fucking Title', :body => "This is some shitty post by a fucking user"}.merge(opts)) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) end def test_it_should_filter_specified_fields p = profane_post p.save assert_equal 'A f*ck*ng Title', p.title - assert_equal 'This is some sh*tty post by a f*ck*ng user', p.body + assert_equal 'This is some f*ck*ng sh*tty-post by a f*ck*ng user', p.body + end + + def test_it_should_handle_nil_fields_bug_9 + p = Post.new({:title => nil, :body => nil}) + p.save + assert_equal nil, p.title + assert_equal nil, p.body + end + + def test_it_should_handle_blank_fields_bug_9 + p = Post.new({:title => "", :body => ""}) + p.save + assert_equal "", p.title + assert_equal "", p.body + end +end + +class VowelsProfanityFilterTest < Test::Unit::TestCase + include VowelsPostHelper + + def profane_post(opts={}) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) + end + + def test_it_should_filter_specified_fields + p = profane_post + p.save + assert_equal 'A F*ck*ng Title', p.title + assert_equal 'This is some f*ck*ng sh*tty-post by a f*ck*ng user', p.body + end + + def test_it_should_handle_nil_fields_bug_9 + p = Post.new({:title => nil, :body => nil}) + p.save + assert_equal nil, p.title + assert_equal nil, p.body + end + + def test_it_should_handle_blank_fields_bug_9 + p = Post.new({:title => "", :body => ""}) + p.save + assert_equal "", p.title + assert_equal "", p.body + end +end + +class HollowProfanityFilterTest < Test::Unit::TestCase + include HollowPostHelper + + def profane_post(opts={}) + Post.new({:title => 'A Fucking Title', :body => "This is some f-u-c-k-i-n-g shitty-post by a fucking user"}.merge(opts)) + end + + def test_it_should_filter_specified_fields + p = profane_post + p.save + assert_equal 'A F*****g Title', p.title + assert_equal 'This is some f*****g s****y-post by a f*****g user', p.body end def test_it_should_handle_nil_fields_bug_9 test/destructive_filter_test.rb @@ -10,9 +10,28 @@ class BasicProfanityFilterTest < Test::Unit::TestCase assert_equal 'happy', ProfanityFilter::Base.clean('happy') end + def test_basic_profanity_filter_does_not_modify_whitespace + assert_equal 'hello world', ProfanityFilter::Base.clean('hello world') + assert_equal "hello \t world", ProfanityFilter::Base.clean("hello \t world") + assert_equal "hello \n world", ProfanityFilter::Base.clean("hello \n world") + end + + def test_basic_profanity_filter_does_not_modify_special_characters + assert_equal 'happy @#$%', ProfanityFilter::Base.clean('happy fuck') + assert_equal 'happy\'s', ProfanityFilter::Base.clean('happy\'s') + assert_equal '@#$%\'s', ProfanityFilter::Base.clean('fuck\'s') + assert_equal '@#$%?!', ProfanityFilter::Base.clean('fuck?!') + end + def test_basic_profanity_filter_replaces_profane_words assert_equal '@#$%', ProfanityFilter::Base.clean('fuck') end + + def test_basic_profanity_filter_replaces_punctuation_spaced_profane_words + assert_equal '@#$%', ProfanityFilter::Base.clean('f-u-c-k') + assert_equal '@#$%', ProfanityFilter::Base.clean('f.u.c.k') + assert_equal 'happy-@#$%', ProfanityFilter::Base.clean('happy-fuck') + end end class DictionaryProfanityFilterTest < Test::Unit::TestCase @@ -20,7 +39,87 @@ class DictionaryProfanityFilterTest < Test::Unit::TestCase assert_equal 'happy', ProfanityFilter::Base.clean('happy', 'dictionary') end + def test_dictionary_profanity_filter_does_not_modify_whitespace + assert_equal 'hello world', ProfanityFilter::Base.clean('hello world', 'dictionary') + assert_equal "hello \t world", ProfanityFilter::Base.clean("hello \t world", 'dictionary') + assert_equal "hello \n world", ProfanityFilter::Base.clean("hello \n world", 'dictionary') + end + + def test_dictionary_profanity_filter_does_not_modify_special_characters + assert_equal 'happy f*ck', ProfanityFilter::Base.clean('happy fuck', 'dictionary') + assert_equal 'happy\'s', ProfanityFilter::Base.clean('happy\'s', 'dictionary') + assert_equal 'f*ck\'s', ProfanityFilter::Base.clean('fuck\'s', 'dictionary') + assert_equal 'f*ck?!', ProfanityFilter::Base.clean('fuck?!', 'dictionary') + end + def test_dictionary_profanity_filter_replaces_profane_words assert_equal 'f*ck', ProfanityFilter::Base.clean('fuck', 'dictionary') end + + def test_dictionary_profanity_filter_replaces_punctuation_spaced_profane_words + assert_equal 'f*ck', ProfanityFilter::Base.clean('f-u-c-k', 'dictionary') + assert_equal 'f*ck', ProfanityFilter::Base.clean('f.u.c.k', 'dictionary') + assert_equal 'happy-f*ck', ProfanityFilter::Base.clean('happy-fuck', 'dictionary') + end +end + + +class VowelsProfanityFilterTest < Test::Unit::TestCase + def test_vowels_profanity_filter_does_not_modify_clean_words + assert_equal 'happy', ProfanityFilter::Base.clean('happy', 'vowels') + end + + def test_vowels_profanity_filter_does_not_modify_whitespace + assert_equal 'hello world', ProfanityFilter::Base.clean('hello world', 'vowels') + assert_equal "hello \t world", ProfanityFilter::Base.clean("hello \t world", 'vowels') + assert_equal "hello \n world", ProfanityFilter::Base.clean("hello \n world", 'vowels') + end + + def test_vowels_profanity_filter_does_not_modify_special_characters + assert_equal 'happy f*ck', ProfanityFilter::Base.clean('happy fuck', 'vowels') + assert_equal 'happy\'s', ProfanityFilter::Base.clean('happy\'s', 'vowels') + assert_equal 'f*ck\'s', ProfanityFilter::Base.clean('fuck\'s', 'vowels') + assert_equal 'f*ck?!', ProfanityFilter::Base.clean('fuck?!', 'vowels') + end + + def test_vowels_profanity_filter_replaces_profane_words + assert_equal 'f*ck', ProfanityFilter::Base.clean('fuck', 'vowels') + assert_equal 'F*CK', ProfanityFilter::Base.clean('FUCK', 'vowels') + end + + def test_vowels_profanity_filter_replaces_punctuation_spaced_profane_words + assert_equal 'f*ck', ProfanityFilter::Base.clean('f-u-c-k', 'vowels') + assert_equal 'f*ck', ProfanityFilter::Base.clean('f.u.c.k', 'vowels') + assert_equal 'happy-f*ck', ProfanityFilter::Base.clean('happy-fuck', 'vowels') + end +end + +class HollowProfanityFilterTest < Test::Unit::TestCase + def test_hollow_profanity_filter_does_not_modify_clean_words + assert_equal 'happy', ProfanityFilter::Base.clean('happy', 'hollow') + end + + def test_hollow_profanity_filter_does_not_modify_whitespace + assert_equal 'hello world', ProfanityFilter::Base.clean('hello world', 'hollow') + assert_equal "hello \t world", ProfanityFilter::Base.clean("hello \t world", 'hollow') + assert_equal "hello \n world", ProfanityFilter::Base.clean("hello \n world", 'hollow') + end + + def test_hollow_profanity_filter_does_not_modify_special_characters + assert_equal 'happy f**k', ProfanityFilter::Base.clean('happy fuck', 'hollow') + assert_equal 'happy\'s', ProfanityFilter::Base.clean('happy\'s', 'hollow') + assert_equal 'f**k\'s', ProfanityFilter::Base.clean('fuck\'s', 'hollow') + assert_equal 'f**k?!', ProfanityFilter::Base.clean('fuck?!', 'hollow') + end + + def test_hollow_profanity_filter_replaces_profane_words + assert_equal 'f**k', ProfanityFilter::Base.clean('fuck', 'hollow') + assert_equal 'F**K', ProfanityFilter::Base.clean('FUCK', 'hollow') + end + + def test_hollow_profanity_filter_replaces_punctuation_spaced_profane_words + assert_equal 'f**k', ProfanityFilter::Base.clean('f-u-c-k', 'hollow') + assert_equal 'f**k', ProfanityFilter::Base.clean('f.u.c.k', 'hollow') + assert_equal 'happy-f**k', ProfanityFilter::Base.clean('happy-fuck', 'hollow') + end end \ No newline at end of file test/profanity_filter_test.rb a5937dba73c09f2806299a609589bc4a654731a3 Neil Ang neil@neilang.com http://github.com/adambair/fu-fu/commit/d7a023a7fafaecb668cd515b73a1c4e762705588 d7a023a7fafaecb668cd515b73a1c4e762705588 2009-08-04T20:25:04-07:00 2009-08-04T20:25:04-07:00 Will now filter punctionation delimited profanity. Added more word replacement options. 69924781a35a9ee353ede393126d3190cfb367c5 Neil Ang neil@neilang.com