Add preserve original token option to ASCIIFolding

Closes #4931
elastic · Feb 14, 2014 · 5c3f4ce · 5c3f4ce
1 parent dffc7cd
commit 5c3f4ce
Show file tree

Hide file tree

Showing 4 changed files with 2,199 additions and 4 deletions.
diff --git a/docs/reference/analysis/tokenfilters/asciifolding-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/asciifolding-tokenfilter.asciidoc
@@ -4,4 +4,43 @@
 A token filter of type `asciifolding` that converts alphabetic, numeric,
 and symbolic Unicode characters which are not in the first 127 ASCII
 characters (the "Basic Latin" Unicode block) into their ASCII
-equivalents, if one exists.
+equivalents, if one exists.  Example:
+
+[source,js]
+--------------------------------------------------
+"index" : {
+    "analysis" : {
+        "analyzer" : {
+            "default" : {
+                "tokenizer" : "standard",
+                "filter" : ["standard", "asciifolding"]
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+coming[1.1.0]
+Accepts `preserve_original` setting which defaults to false but if true
+will keep the oringal token as well as emit the folded token.  For
+example:
+
+[source,js]
+--------------------------------------------------
+"index" : {
+    "analysis" : {
+        "analyzer" : {
+            "default" : {
+                "tokenizer" : "standard",
+                "filter" : ["standard", "my_ascii_folding"]
+            }
+        },
+        "filter" : {
+            "my_ascii_folding" : {
+                "type" : "asciifolding",
+                "preserve_original" : true
+            }
+        }
+    }
+}
+--------------------------------------------------