updated regexes to handle utf-8 correctly

jfahrenkrug · Feb 13, 2013 · 8be024d · 8be024d
1 parent e05633f
commit 8be024d
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/lib/xapit/indexers/abstract_indexer.rb b/lib/xapit/indexers/abstract_indexer.rb
@@ -77,7 +77,7 @@ def text_terms(member) # REFACTORME some duplicaiton with simple indexer
         if options[:proc]
           options[:proc].call(content).reject(&:blank?).map(&:to_s).map(&:xapit_utf8_downcase)
         else
-          content.scan(/\w+/u).map(&:xapit_utf8_downcase)
+          content.scan(/\p{Word}+/u).map(&:xapit_utf8_downcase)
         end
       end.flatten
     end

diff --git a/lib/xapit/indexers/simple_indexer.rb b/lib/xapit/indexers/simple_indexer.rb
@@ -27,7 +27,7 @@ def terms_for_attribute(member, name, options)
       elsif content.kind_of? Array
         content.reject(&:blank?).map(&:to_s).map(&:xapit_utf8_downcase)
       else
-        content.to_s.scan(/\w+/u).map(&:xapit_utf8_downcase)
+        content.to_s.scan(/\p{Word}+/u).map(&:xapit_utf8_downcase)
       end
     end
 

diff --git a/lib/xapit/query_parsers/abstract_query_parser.rb b/lib/xapit/query_parsers/abstract_query_parser.rb
@@ -96,11 +96,11 @@ def facet_identifiers
 
     def spelling_suggestion
       raise "Spelling has been disabled. Enable spelling in Xapit.setup." unless Config.spelling?
-      if [@search_text, *@search_text.scan(/\w+/)].all? { |term| term_suggestion(term).nil? }
+      if [@search_text, *@search_text.scan(/\p{Word}+/)].all? { |term| term_suggestion(term).nil? }
         nil
       else
         return term_suggestion(@search_text) unless term_suggestion(@search_text).blank?
-        @search_text.xapit_utf8_downcase.gsub(/\w+/) do |term|
+        @search_text.xapit_utf8_downcase.gsub(/\p{Word}+/) do |term|
           term_suggestion(term) || term
         end
       end