Suggester fixes

src/Nest/Nest.csproj

@@ -1261,6 +1261,11 @@
     <Compile Include="Search\Suggesters\PhraseSuggester\PhraseSuggestCollate.cs" />
     <Compile Include="Search\Suggesters\PhraseSuggester\PhraseSuggester.cs" />
     <Compile Include="Search\Suggesters\PhraseSuggester\PhraseSuggestHighlight.cs" />
+    <Compile Include="Search\Suggesters\PhraseSuggester\SmoothingModel\LinearInterpolationSmoothingModel.cs" />
+    <Compile Include="Search\Suggesters\PhraseSuggester\SmoothingModel\LaplaceSmoothingModel.cs" />
+    <Compile Include="Search\Suggesters\PhraseSuggester\SmoothingModel\StupidBackoffSmoothingModel.cs" />
+    <Compile Include="Search\Suggesters\PhraseSuggester\SmoothingModel\SmoothingModelContainer.cs" />
+    <Compile Include="Search\Suggesters\PhraseSuggester\SmoothingModel\SmoothingModelBase.cs" />
     <Compile Include="Search\Suggesters\Suggest.cs" />
     <Compile Include="Search\Suggesters\SuggestBucket.cs" />
     <Compile Include="Search\Suggesters\SuggestContainer.cs" />

src/Nest/Search/Suggesters/PhraseSuggester/PhraseSuggester.cs

@@ -38,6 +38,9 @@ public interface IPhraseSuggester : ISuggester
 
 		[JsonProperty("collate")]
 		IPhraseSuggestCollate Collate { get; set; }
+
+		[JsonProperty("smoothing")]
+		SmoothingModelContainer Smoothing { get; set; }
 	}
 
 	public class PhraseSuggester : SuggesterBase, IPhraseSuggester
@@ -52,6 +55,7 @@ public class PhraseSuggester : SuggesterBase, IPhraseSuggester
 		public IEnumerable<IDirectGenerator> DirectGenerator { get; set; }
 		public IPhraseSuggestHighlight Highlight { get; set; }
 		public IPhraseSuggestCollate Collate { get; set; }
+		public SmoothingModelContainer Smoothing { get; set; }
 	}
 
 	public class PhraseSuggesterDescriptor<T> : SuggestDescriptorBase<PhraseSuggesterDescriptor<T>, IPhraseSuggester, T>, IPhraseSuggester
@@ -67,6 +71,7 @@ public class PhraseSuggesterDescriptor<T> : SuggestDescriptorBase<PhraseSuggeste
 		IEnumerable<IDirectGenerator> IPhraseSuggester.DirectGenerator { get; set; }
 		IPhraseSuggestHighlight IPhraseSuggester.Highlight { get; set; }
 		IPhraseSuggestCollate IPhraseSuggester.Collate { get; set; }
+		SmoothingModelContainer IPhraseSuggester.Smoothing { get; set; }
 
 		public PhraseSuggesterDescriptor<T> Text(string text) => Assign(a => a.Text = text);
 
@@ -91,5 +96,8 @@ public PhraseSuggesterDescriptor<T> Highlight(Func<PhraseSuggestHighlightDescrip
 
 		public PhraseSuggesterDescriptor<T> Collate(Func<PhraseSuggestCollateDescriptor<T>, IPhraseSuggestCollate> selector) =>
 			Assign(a => a.Collate = selector?.Invoke(new PhraseSuggestCollateDescriptor<T>()));
+
+		public PhraseSuggesterDescriptor<T> Smoothing(Func<SmoothingModelContainerDescriptor, SmoothingModelContainer> selector) =>
+			Assign(a => a.Smoothing = selector?.Invoke(new SmoothingModelContainerDescriptor()));
 	}
 }

src/Nest/Search/Suggesters/PhraseSuggester/SmoothingModel/LaplaceSmoothingModel.cs

@@ -0,0 +1,27 @@
+using Newtonsoft.Json;
+using System;
+
+namespace Nest
+{
+	[JsonObject]
+	[JsonConverter(typeof(ReadAsTypeJsonConverter<LaplaceSmoothingModel>))]
+	public interface ILaplaceSmoothingModel : ISmoothingModel
+	{
+		[JsonProperty("alpha")]
+		double? Alpha { get; set; }
+	}
+
+	public class LaplaceSmoothingModel : SmoothingModelBase, ILaplaceSmoothingModel
+	{
+		public double? Alpha { get; set; }
+
+		internal override void WrapInContainer(ISmoothingModelContainer container) => container.Laplace = this;
+	}
+
+	public class LaplaceSmoothingModelDescriptor : DescriptorBase<LaplaceSmoothingModelDescriptor, ILaplaceSmoothingModel>, ILaplaceSmoothingModel
+	{
+		double? ILaplaceSmoothingModel.Alpha{ get; set; }
+
+		public LaplaceSmoothingModelDescriptor Alpha(double? alpha) => Assign(a => a.Alpha = alpha);
+	}
+}

src/Nest/Search/Suggesters/PhraseSuggester/SmoothingModel/LinearInterpolationSmoothingModel.cs

@@ -0,0 +1,37 @@
+using Newtonsoft.Json;
+using System;
+
+namespace Nest
+{
+	[JsonObject]
+	[JsonConverter(typeof(ReadAsTypeJsonConverter<LinearInterpolationSmoothingModel>))]
+	public interface ILinearInterpolationSmoothingModel : ISmoothingModel
+	{
+		[JsonProperty("trigram_lambda")]
+		double TrigramLambda { get; set; }
+		[JsonProperty("bigram_lambda")]
+		double BigramLambda { get; set; }
+		[JsonProperty("unigram_lambda")]
+		double UnigramLambda { get; set; }
+	}
+
+	public class LinearInterpolationSmoothingModel : SmoothingModelBase, ILinearInterpolationSmoothingModel
+	{
+		public double TrigramLambda { get; set; }
+		public double BigramLambda { get; set; }
+		public double UnigramLambda { get; set; }
+
+		internal override void WrapInContainer(ISmoothingModelContainer container) => container.LinearInterpolation = this;
+	}
+
+	public class LinearInterpolationSmoothingModelDescriptor : DescriptorBase<LinearInterpolationSmoothingModelDescriptor, ILinearInterpolationSmoothingModel>, ILinearInterpolationSmoothingModel
+	{
+		double ILinearInterpolationSmoothingModel.TrigramLambda { get; set; }
+		double ILinearInterpolationSmoothingModel.UnigramLambda { get; set; }
+		double ILinearInterpolationSmoothingModel.BigramLambda { get; set; }
+
+		public LinearInterpolationSmoothingModelDescriptor TrigramLambda(double lambda) => Assign(a => a.TrigramLambda = lambda);
+		public LinearInterpolationSmoothingModelDescriptor UnigramLambda(double lambda) => Assign(a => a.UnigramLambda = lambda);
+		public LinearInterpolationSmoothingModelDescriptor BigramLambda(double lambda) => Assign(a => a.BigramLambda = lambda);
+	}
+}

src/Nest/Search/Suggesters/PhraseSuggester/SmoothingModel/SmoothingModelBase.cs

@@ -0,0 +1,16 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject]
+	public interface ISmoothingModel {}
+
+	public abstract class SmoothingModelBase
+	{
+		public static implicit operator SmoothingModelContainer(SmoothingModelBase model) => model == null
+			? null
+			: new SmoothingModelContainer(model);
+
+		internal abstract void WrapInContainer(ISmoothingModelContainer container);
+	}
+}

src/Nest/Search/Suggesters/PhraseSuggester/SmoothingModel/SmoothingModelContainer.cs

@@ -0,0 +1,49 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject(MemberSerialization.OptIn)]
+	[JsonConverter(typeof(ReserializeJsonConverter<SmoothingModelContainer, ISmoothingModelContainer>))]
+	public interface ISmoothingModelContainer
+	{
+		[JsonProperty("stupid_backoff")]
+		IStupidBackoffSmoothingModel StupidBackoff { get; set; }
+
+		[JsonProperty("laplace")]
+		ILaplaceSmoothingModel Laplace { get; set; }
+
+		[JsonProperty("linear_interpolation")]
+		ILinearInterpolationSmoothingModel LinearInterpolation { get; set; }
+	}
+
+	[JsonObject(MemberSerialization.OptIn)]
+	public class SmoothingModelContainer : ISmoothingModelContainer, IDescriptor
+	{
+		internal SmoothingModelContainer() {}
+
+		public SmoothingModelContainer(SmoothingModelBase model)
+		{
+			model.ThrowIfNull(nameof(model));
+			model.WrapInContainer(this);
+		}
+
+		IStupidBackoffSmoothingModel ISmoothingModelContainer.StupidBackoff { get; set; }
+		ILaplaceSmoothingModel ISmoothingModelContainer.Laplace { get; set; }
+		ILinearInterpolationSmoothingModel ISmoothingModelContainer.LinearInterpolation { get; set; }
+	}
+
+	public class SmoothingModelContainerDescriptor : SmoothingModelContainer
+	{
+		private SmoothingModelContainerDescriptor Assign(Action<ISmoothingModelContainer> assigner) => Fluent.Assign(this, assigner);
+
+		public SmoothingModelContainerDescriptor StupidBackoff(Func<StupidBackoffSmoothingModelDescriptor, IStupidBackoffSmoothingModel> selector) =>
+			Assign(a => a.StupidBackoff = selector?.InvokeOrDefault(new StupidBackoffSmoothingModelDescriptor()));
+
+		public SmoothingModelContainerDescriptor LinearInterpolation(Func<LinearInterpolationSmoothingModelDescriptor, ILinearInterpolationSmoothingModel> selector) =>
+			Assign(a => a.LinearInterpolation = selector?.InvokeOrDefault(new LinearInterpolationSmoothingModelDescriptor()));
+
+		public SmoothingModelContainerDescriptor Laplace(Func<LaplaceSmoothingModelDescriptor, ILaplaceSmoothingModel> selector) =>
+			Assign(a => a.Laplace = selector?.InvokeOrDefault(new LaplaceSmoothingModelDescriptor()));
+	}
+}

src/Nest/Search/Suggesters/PhraseSuggester/SmoothingModel/StupidBackoffSmoothingModel.cs

@@ -0,0 +1,27 @@
+using Newtonsoft.Json;
+using System;
+
+namespace Nest
+{
+	[JsonObject]
+	[JsonConverter(typeof(ReadAsTypeJsonConverter<StupidBackoffSmoothingModel>))]
+	public interface IStupidBackoffSmoothingModel : ISmoothingModel
+	{
+		[JsonProperty("discount")]
+		double? Discount { get; set; }
+	}
+
+	public class StupidBackoffSmoothingModel : SmoothingModelBase, IStupidBackoffSmoothingModel
+	{
+		public double? Discount { get; set; }
+
+		internal override void WrapInContainer(ISmoothingModelContainer container) => container.StupidBackoff = this;
+	}
+
+	public class StupidBackoffSmoothingModelDescriptor : DescriptorBase<StupidBackoffSmoothingModelDescriptor, IStupidBackoffSmoothingModel>, IStupidBackoffSmoothingModel
+	{
+		double? IStupidBackoffSmoothingModel.Discount{ get; set; }
+
+		public StupidBackoffSmoothingModelDescriptor Discount(double? discount) => Assign(a => a.Discount = discount);
+	}
+}

src/Nest/Search/Suggesters/Suggest/SuggestResponseJsonConverter.cs

@@ -25,7 +25,6 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 			var o = serializer.Deserialize(jsonObject.CreateReader(), genericType);
 			var suggestType = typeof(Suggest<>).MakeGenericType(genericType).MakeArrayType();
 
-
 			var dict = new Dictionary<string, object>();
 
 			foreach (var prop in jsonObject.Properties())
@@ -36,6 +35,5 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 			var r = typeof(SuggestResponse<>).CreateGenericInstance(genericType, shards, dict);
 			return r;
 		}
-
 	}
 }

src/Nest/Search/Suggesters/SuggestOption.cs

@@ -13,26 +13,59 @@ public class SuggestOption<T> where T : class
 		public string Text { get; internal set; }
 
 		[JsonProperty("_score")]
-		public double Score { get; internal set; }
+		internal double? DocumentScore { get; set; }
 
+		[JsonProperty("score")]
+		internal double? SuggestScore { get; set; }
+
+		[JsonIgnore]
+		public double Score => DocumentScore ?? SuggestScore ?? 0;
+
+		/// <summary>
+		/// Term suggester only
+		/// </summary>
+		[JsonProperty("freq")]
+		public long Frequency { get; set; }
+
+		/// <summary>
+		/// Completion suggester only, the index of the completed document
+		/// </summary>
 		[JsonProperty("_index")]
 		public IndexName Index { get; internal set; }
 
+		/// <summary>
+		/// Completion suggester only, the type of the completed document
+		/// </summary>
 		[JsonProperty("_type")]
 		public TypeName Type { get; internal set; }
 
+		/// <summary>
+		/// Completion suggester only, the id of the completed document
+		/// </summary>
 		[JsonProperty("_id")]
 		public Id Id { get; internal set; }
 
+		/// <summary>
+		/// Completion suggester only, the source of the completed document
+		/// </summary>
 		[JsonProperty("_source")]
 		public T Source { get; internal set; }
 
+		/// <summary>
+		/// Completion suggester only, the contexts associated with the completed document
+		/// </summary>
 		[JsonProperty("contexts")]
 		public IDictionary<string, IEnumerable<Context>> Contexts { get; internal set; }
 
+		/// <summary>
+		/// Phrase suggester only, higlighted version of text
+		/// </summary>
 		[JsonProperty("highlighted")]
 		public string Highlighted { get; internal set; }
 
+		/// <summary>
+		/// Phrase suggestions only, true if matching documents for the collate query were found,
+		/// </summary>
 		[JsonProperty("collate_match")]
 		public bool CollateMatch { get; internal set; }
 

src/Tests/Framework/EndpointTests/Bootstrappers/Seeder.cs

@@ -84,23 +84,6 @@ private void CreateIndexTemplate()
 					.NumberOfReplicas(0)
 					.NumberOfShards(2)
 				)
-				//.Mappings(pm => pm
-				//	.Map("_default_", m => m
-				//		.DynamicTemplates(dt => dt
-				//			.DynamicTemplate("raw_field", dtt => dtt
-				//				.Match("*") //matches all fields
-				//				.MatchMappingType("string") //that are a string
-				//				.Mapping(tm => tm
-				//					.Text(sm => sm //map as string
-				//						.Fields(f => f //with a multifield 'raw' that is not analyzed
-				//							.Keyword(ssm => ssm.Name("raw"))
-				//						)
-				//					)
-				//				)
-				//			)
-				//		)
-				//	)
-				//)
 			);
 			putTemplateResult.IsValid.Should().BeTrue();
 		}
@@ -121,6 +104,9 @@ private void CreateDeveloperIndex()
 		private void CreateProjectIndex()
 		{
 			var createProjectIndex = this.Client.CreateIndex(typeof(Project), c => c
+				.Settings(settings=>settings
+					.Analysis(ProjectAnalysisSettings)
+				)
 				.Aliases(a => a
 					.Alias("projects-alias")
 				)
@@ -149,6 +135,21 @@ private void CreateProjectIndex()
 			createProjectIndex.IsValid.Should().BeTrue();
 		}
 
+		private IAnalysis ProjectAnalysisSettings(AnalysisDescriptor analysis) => analysis
+			.TokenFilters(tokenFilters => tokenFilters
+				.Shingle("shingle", shingle=>shingle
+					.MinShingleSize(2)
+					.MaxShingleSize(4)
+				)
+			)
+			.Analyzers(analyzers=>analyzers
+				.Custom("shingle", shingle=>shingle
+					.Filters("standard", "shingle")
+					.Tokenizer("standard")
+				)
+			);
+
+
 		private void CreatePercolatorIndex()
 		{
 			var createPercolatedIndex = this.Client.CreateIndex(typeof(PercolatedQuery), c => c
@@ -183,6 +184,12 @@ public static PropertiesDescriptor<Project> ProjectProperties(PropertiesDescript
 				.Text(s=>s
 					.Name(p=>p.Description)
 					.Fielddata()
+					.Fields(f=>f
+						.Text(t=>t
+							.Name("shingle")
+							.Analyzer("shingle")
+						)
+					)
 				)
 				.Date(d => d
 					.Store()