From 156d4bbafa77eb7147e8873f2d5704e8e78d2173 Mon Sep 17 00:00:00 2001
From: Martijn Laarman <Mpdreamz@gmail.com>
Date: Wed, 17 Oct 2018 04:59:42 +0200
Subject: [PATCH] Add Multiplexer Token Filter (#3425)

(cherry picked from commit ef88c305eded40b43bba0124aba249bf81def593)
---
 .../TokenFilters/MultiplexerTokenFilter.cs    | 50 +++++++++++++++++++
 .../Analysis/TokenFilters/TokenFilters.cs     |  7 +++
 .../Indices/Analyze/AnalyzeTokenFilters.cs    |  4 ++
 .../Analysis/TokenFilters/TokenFilterTests.cs | 26 +++++++++-
 4 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 src/Nest/Analysis/TokenFilters/MultiplexerTokenFilter.cs

diff --git a/src/Nest/Analysis/TokenFilters/MultiplexerTokenFilter.cs b/src/Nest/Analysis/TokenFilters/MultiplexerTokenFilter.cs
new file mode 100644
index 00000000000..9454b6ad78a
--- /dev/null
+++ b/src/Nest/Analysis/TokenFilters/MultiplexerTokenFilter.cs
@@ -0,0 +1,50 @@
+using System.Collections;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// A token filter of type multiplexer will emit multiple tokens at the same position, each version of the token having
+	/// been run through a different filter. Identical output tokens at the same position will be removed.
+	/// </summary>
+	public interface IMultiplexerTokenFilter : ITokenFilter
+	{
+		[JsonProperty("filters")]
+		IEnumerable<string> Filters { get; set; }
+
+		[JsonProperty("preserve_original")]
+		bool? PreserveOriginal { get; set; }
+	}
+	public class MultiplexerTokenFilter : TokenFilterBase, IMultiplexerTokenFilter
+	{
+		internal const string TokenType = "multiplexer";
+		public MultiplexerTokenFilter() : base(TokenType) { }
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.Filters"/>
+		public IEnumerable<string> Filters { get; set; }
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.PreserveOriginal"/>
+		public bool? PreserveOriginal { get; set; }
+
+	}
+	///<inheritdoc cref="IMultiplexerTokenFilter"/>
+	public class MultiplexerTokenFilterDescriptor
+		: TokenFilterDescriptorBase<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter>, IMultiplexerTokenFilter
+	{
+		protected override string Type => MultiplexerTokenFilter.TokenType;
+
+		IEnumerable<string> IMultiplexerTokenFilter.Filters { get; set; }
+		bool? IMultiplexerTokenFilter.PreserveOriginal { get; set; }
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.Filters"/>
+		public MultiplexerTokenFilterDescriptor Filters(IEnumerable<string> filters) => Assign(a => a.Filters = filters);
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.Filters"/>
+		public MultiplexerTokenFilterDescriptor Filters(params string[] filters) => Assign(a => a.Filters = filters);
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.PreserveOriginal"/>
+		public MultiplexerTokenFilterDescriptor PreserveOriginal(bool? preserve = true) => Assign(a => a.PreserveOriginal = preserve);
+	}
+
+}
diff --git a/src/Nest/Analysis/TokenFilters/TokenFilters.cs b/src/Nest/Analysis/TokenFilters/TokenFilters.cs
index d44f4989acb..2b9fac67c65 100644
--- a/src/Nest/Analysis/TokenFilters/TokenFilters.cs
+++ b/src/Nest/Analysis/TokenFilters/TokenFilters.cs
@@ -317,5 +317,12 @@ public TokenFiltersDescriptor() : base(new TokenFilters())
 		/// </summary>
 		public TokenFiltersDescriptor IcuTransform(string name, Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
 			Assign(name, selector.Invoke(new IcuTransformTokenFilterDescriptor()));
+
+		/// <summary>
+		///  A token filter of type multiplexer will emit multiple tokens at the same position, each version of the token
+		/// having been run through a different filter. Identical output tokens at the same position will be removed.
+		/// </summary>
+		public TokenFiltersDescriptor Multiplexer(string name, Func<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter> selector) =>
+			Assign(name, selector.Invoke(new MultiplexerTokenFilterDescriptor()));
 	}
 }
diff --git a/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs b/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs
index 17b4fd93ff8..4c9ad908d84 100644
--- a/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs
+++ b/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs
@@ -323,6 +323,10 @@ public class AnalyzeTokenFiltersDescriptor : DescriptorPromiseBase<AnalyzeTokenF
 		/// </summary>
 		public AnalyzeTokenFiltersDescriptor IcuTransform(Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
 			AssignIfNotNull(selector.Invoke(new IcuTransformTokenFilterDescriptor()));
+
+		///<inheritdoc cref="IMultiplexerTokenFilter"/>
+		public AnalyzeTokenFiltersDescriptor Multiplexer(Func<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter> selector) =>
+			AssignIfNotNull(selector.Invoke(new MultiplexerTokenFilterDescriptor()));
 	}
 
 }
diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs
index 4c702cf10d1..b21fcdb4f8a 100644
--- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs
+++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs
@@ -1,4 +1,5 @@
-using System;
+using System;
+using Elastic.Xunit.XunitPlumbing;
 using Nest;
 using Tests.Framework;
 
@@ -878,5 +879,28 @@ public class PhoneticTests : TokenFilterAssertionBase<PhoneticTests>
 			};
 
 		}
+
+		[SkipVersion("<6.4.0", "Introduced in 6.4.0")]
+		public class MultiplexerTests : TokenFilterAssertionBase<PhoneticTests>
+		{
+			public override string Name => "multiplexer";
+			public override object Json => new
+			{
+				filters = new[]{"lowercase", "lowercase, porter_stem"},
+				preserve_original = true
+			};
+
+			public override ITokenFilter Initializer => new MultiplexerTokenFilter
+			{
+				Filters = new[] {"lowercase", "lowercase, porter_stem"},
+				PreserveOriginal = true
+			};
+
+			public override FuncTokenFilters Fluent => (n, tf) => tf
+				.Multiplexer(n, t => t
+					.Filters("lowercase", "lowercase, porter_stem")
+					.PreserveOriginal()
+				);
+		}
 	}
 }