Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Already on GitHub? Sign in to your account

Use Unicode properties in generators #78

Merged
merged 1 commit into from Apr 26, 2012
Jump to file or symbol
Failed to load files and symbols.
+6 −6
Split
@@ -44,7 +44,7 @@ def each
#
books_index = Index.new :books do
source { Books.new }
- indexing removes_characters: /[^a-z0-9\s\/\-\_\:\"\&\.]/i,
+ indexing removes_characters: /[^\p{L}\p{N}\s\/\-\_\:\"\&\.]/i,
stopwords: /\b(and|the|of|it|in|for)\b/i,
splits_text_on: /[\s\/\-\_\:\"\&\/]/
category :title,
@@ -64,7 +64,7 @@ def each
#
books = Search.new books_index do
searching substitutes_characters_with: CharacterSubstituters::WestEuropean.new, # Normalizes special user input, Ä -> Ae, ñ -> n etc.
- removes_characters: /[^a-z0-9\s\/\-\_\&\.\"\~\*\:\,]/i, # Picky needs control chars *"~:, to pass through.
+ removes_characters: /[^\p{L}\p{N}\s\/\-\_\&\.\"\~\*\:\,]/i, # Picky needs control chars *"~:, to pass through.
stopwords: /\b(and|the|of|it|in|for)\b/i,
splits_text_on: /[\s\/\-\&]+/
@@ -129,4 +129,4 @@ def js path
end
-end
+end
@@ -31,7 +31,7 @@ def each
#
books_index = Index.new :books do
source { Books.new }
- indexing removes_characters: /[^a-z0-9\s\/\-\_\:\"\&\.]/i,
+ indexing removes_characters: /[^\p{L}\p{N}\s\/\-\_\:\"\&\.]/i,
stopwords: /\b(and|the|of|it|in|for)\b/i,
splits_text_on: /[\s\/\-\_\:\"\&\/]/
category :title,
@@ -51,7 +51,7 @@ def each
#
books = Search.new books_index do
searching substitutes_characters_with: CharacterSubstituters::WestEuropean.new, # Normalizes special user input, Ä -> Ae, ñ -> n etc.
- removes_characters: /[^a-z0-9\s\/\-\_\&\.\"\~\*\:\,]/i, # Picky needs control chars *"~:, to pass through.
+ removes_characters: /[^\p{L}\p{N}\s\/\-\_\&\.\"\~\*\:\,]/i, # Picky needs control chars *"~:, to pass through.
stopwords: /\b(and|the|of|it|in|for)\b/i,
splits_text_on: /[\s\/\-\&]+/
boost [:title, :author] => +3,
@@ -66,4 +66,4 @@ def each
results.to_json
end
-end
+end