add tests for tokenized_by scope

lib/acts_as_tokenizable/acts_as_tokenizable.rb

@@ -51,14 +51,14 @@ module TokenizedBy
       scope :tokenized_by, lambda { |search_token|
         search_strings = []
         search_values = []
-        StringUtils.words(prepare_search_token(search_token)).each do |w|
+        StringUtils.words(search_token).each do |w|
           if w[0, 1] == '-'
             search_strings.push("#{table_name}.#{token_field_name} NOT LIKE ?")
-            search_values.push("%#{w[1, w.length]}%")
           else
             search_strings.push("#{table_name}.#{token_field_name} LIKE ?")
-            search_values.push("%#{w}%")
           end
+          tokenized_word = StringUtils.to_token(w)
+          search_values.push("%#{tokenized_word}%")
         end
         where([search_strings.join(' AND '), *search_values])
       }

lib/acts_as_tokenizable/string_utils.rb

@@ -50,7 +50,7 @@ def self.alphanumerics(str)
     def self.to_token(str, max_length = 255)
       # to_slug and normalize are provided by the 'babosa' gem
       # remove all non-alphanumeric but hyphen (-)
-      str = str.to_slug.normalize.strip.downcase.gsub(/[^\w|-]/, '')
+      str = str.to_slug.normalize.strip.downcase.gsub(/[\s|\.|,]+/, '')
       # remove duplicates, except on pure numbers
       str = str.squeeze unless numeric?(str)
       str[0..(max_length - 1)]

spec/acts_as_tokenizable_spec.rb

@@ -3,25 +3,63 @@
 
 describe ActiveRecord do
   describe 'classes with acts_as_tokenizable' do
-    it 'does include to_token and tokenize methods' do
-      expect(Friend.instance_methods)
-        .to include(:tokenize, :tokenize!, :to_token)
+    it 'respond to `tokenized_by`' do
+      expect(Friend.methods).to include(:tokenized_by)
     end
 
-    it 'updates the token field after creating an object' do
-      friend = Friend.new name: 'John', email: 'john@example.com'
-      expect(friend.token).to be_nil
-      friend.save
-      expect(friend.token).to eq('john')
+    it 'raises an error if `to_token` method is not defined' do
+      expect { Enemy.to_token }.to raise_error(NoMethodError)
     end
 
-    it 'allows different token field name' do
-      malmo = City.create name: 'Malmö'
-      expect(malmo.tokenized_name).to eq('malmo')
+    describe 'instances' do
+      it 'include to_token and tokenize methods' do
+        expect(Friend.instance_methods)
+          .to include(:tokenize, :tokenize!, :to_token)
+      end
+
+      it 'update the token field after creating an object' do
+        friend = Friend.new name: 'John', email: 'john@example.com', age: 30
+        expect(friend.token).to be_nil
+        friend.save
+        expect(friend.token).to eq('john 30')
+      end
+
+      it 'allow different token field name' do
+        malmo = City.create name: 'Malmö'
+        expect(malmo.tokenized_name).to eq('malmo')
+      end
     end
 
-    it 'raises an error if `to_token` method is not defined' do
-      expect { Enemy.to_token }.to raise_error(NoMethodError)
+    describe 'scope tokenized_by' do
+      before do
+        Friend.delete_all
+
+        Friend.create name: 'Tomás', age: 31
+        Friend.create name: 'Rafa', age: 25
+        Friend.create name: 'Matthias', age: 35
+        Friend.create name: 'Mamá', age: 30
+      end
+
+      it 'finds records with tokenized string' do
+        expect(Friend.tokenized_by('as').count).to eq(2)
+      end
+
+      it 'finds records with tokenized with number' do
+        expect(Friend.tokenized_by('3').count).to eq(3)
+      end
+
+      it 'finds records with duplicated characters' do
+        expect(Friend.tokenized_by('mathias').count).to eq(1)
+        expect(Friend.tokenized_by('matthias').count).to eq(1)
+      end
+
+      it 'excludes records with negative token' do
+        expect(Friend.tokenized_by('-To').count).to eq(3)
+      end
+
+      it 'combines tokens' do
+        expect(Friend.tokenized_by('-To 5').count).to eq(2)
+      end
     end
   end
 

spec/support_helper.rb

@@ -9,7 +9,8 @@ class Friend < ActiveRecord::Base
   acts_as_tokenizable :token
 
   def to_token
-    ActsAsTokenizable::StringUtils.words_to_token(name)
+    string_to_token = [name, age].join(' ')
+    ActsAsTokenizable::StringUtils.words_to_token(string_to_token)
   end
 end