From 9fb1620c01a6e120fcd3ea405da8e74b4f37945a Mon Sep 17 00:00:00 2001 From: Greg Marzouka Date: Thu, 4 Sep 2014 17:24:27 -0400 Subject: [PATCH 1/3] Partial implementation of the top hits aggregation TODO: Still need to implement a top hits response object and handle deserialization in the AggregationConverter class. --- .../DSL/Aggregations/AggregationDescriptor.cs | 21 ++++- .../TopHitsAggregationDescriptor.cs | 91 +++++++++++++++++++ src/Nest/Nest.csproj | 1 + .../Aggregations/MetricAggregationTests.cs | 26 ++++++ 4 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 src/Nest/DSL/Aggregations/TopHitsAggregationDescriptor.cs diff --git a/src/Nest/DSL/Aggregations/AggregationDescriptor.cs b/src/Nest/DSL/Aggregations/AggregationDescriptor.cs index e53c805623e..3df2ad0d03c 100644 --- a/src/Nest/DSL/Aggregations/AggregationDescriptor.cs +++ b/src/Nest/DSL/Aggregations/AggregationDescriptor.cs @@ -75,6 +75,9 @@ public interface IAggregationContainer [JsonProperty("value_count")] IValueCountAggregator ValueCount { get; set; } + [JsonProperty("top_hits")] + ITopHitsAggregator TopHits { get; set; } + [JsonProperty("aggs")] [JsonConverter(typeof(DictionaryKeysAreNotPropertyNamesJsonConverter))] IDictionary Aggregations { get; set; } @@ -98,6 +101,7 @@ public class AggregationContainer : IAggregationContainer private IRangeAggregator _range; private ITermsAggregator _terms; private ISignificantTermsAggregator _significantTerms; + private ITopHitsAggregator _topHits; public IAverageAggregator Average { get; set; } public IValueCountAggregator ValueCount { get; set; } public IMaxAggregator Max { get; set; } @@ -105,7 +109,6 @@ public class AggregationContainer : IAggregationContainer public IStatsAggregator Stats { get; set; } public ISumAggregator Sum { get; set; } public IExtendedStatsAggregator ExtendedStats { get; set; } - public IDateHistogramAggregator DateHistogram { get { return _dateHistogram; } @@ -196,6 +199,12 @@ public ISignificantTermsAggregator SignificantTerms set { _significantTerms = value; } } + public ITopHitsAggregator TopHits + { + get { return _topHits; } + set { _topHits = value; } + } + private void LiftAggregations(IBucketAggregator bucket) { if (bucket == null) return; @@ -253,7 +262,9 @@ public class AggregationDescriptor : IAggregationContainer ISignificantTermsAggregator IAggregationContainer.SignificantTerms { get; set; } ITermsAggregator IAggregationContainer.Terms { get; set; } - + + ITopHitsAggregator IAggregationContainer.TopHits { get; set; } + public AggregationDescriptor Average(string name, Func, AverageAggregationDescriptor> selector) { return _SetInnerAggregation(name, selector, (a, d) => a.Average = d); @@ -375,6 +386,12 @@ public AggregationDescriptor ValueCount(string name, return _SetInnerAggregation(name, selector, (a, d) => a.ValueCount = d); } + public AggregationDescriptor TopHits(string name, + Func, TopHitsAggregationDescriptor> selector) + { + return _SetInnerAggregation(name, selector, (a, d) => a.TopHits = d); + } + private AggregationDescriptor _SetInnerAggregation( string key, Func selector diff --git a/src/Nest/DSL/Aggregations/TopHitsAggregationDescriptor.cs b/src/Nest/DSL/Aggregations/TopHitsAggregationDescriptor.cs new file mode 100644 index 00000000000..b86ccc99207 --- /dev/null +++ b/src/Nest/DSL/Aggregations/TopHitsAggregationDescriptor.cs @@ -0,0 +1,91 @@ +using Nest.Resolvers.Converters; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Nest +{ + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] + [JsonConverter(typeof(ReadAsTypeConverter))] + public interface ITopHitsAggregator : IMetricAggregator + { + [JsonProperty("from")] + int? From { get; set; } + + [JsonProperty("size")] + int? Size { get; set; } + + [JsonProperty("sort")] + [JsonConverter(typeof(SortCollectionConverter))] + IList> Sort { get; set; } + + [JsonProperty("_source")] + ISourceFilter Source { get; set; } + } + + public class TopHitsAggregator : MetricAggregator, ITopHitsAggregator + { + public int? From { get; set; } + public int? Size { get; set; } + public IList> Sort { get; set; } + public ISourceFilter Source { get; set; } + } + + public class TopHitsAggregationDescriptor + : MetricAggregationBaseDescriptor, T>, ITopHitsAggregator + where T : class + { + ITopHitsAggregator Self { get { return this; } } + + int? ITopHitsAggregator.From { get; set; } + + int? ITopHitsAggregator.Size { get; set; } + + IList> ITopHitsAggregator.Sort { get; set; } + + ISourceFilter ITopHitsAggregator.Source { get; set; } + + public TopHitsAggregationDescriptor From(int from) + { + this.Self.From = from; + return this; + } + + public TopHitsAggregationDescriptor Size(int size) + { + this.Self.Size = size; + return this; + } + + public TopHitsAggregationDescriptor Sort(Func, IFieldSort> sortSelector) + { + sortSelector.ThrowIfNull("sortSelector"); + + if (Self.Sort == null) + Self.Sort = new List>(); + + var descriptor = sortSelector(new SortFieldDescriptor()); + this.Self.Sort.Add(new KeyValuePair(descriptor.Field, descriptor)); + + return this; + } + + public TopHitsAggregationDescriptor Source(bool include = true) + { + if (!include) + this.Self.Source = new SourceFilter { Exclude = new PropertyPathMarker[] { "*" } }; + else + this.Self.Source = null; + + return this; + } + + public TopHitsAggregationDescriptor Source(Func, SearchSourceDescriptor> sourceSelector) + { + this.Self.Source = sourceSelector(new SearchSourceDescriptor()); + return this; + } + } +} diff --git a/src/Nest/Nest.csproj b/src/Nest/Nest.csproj index 0a0d96688ce..aac313f38a1 100644 --- a/src/Nest/Nest.csproj +++ b/src/Nest/Nest.csproj @@ -207,6 +207,7 @@ + diff --git a/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs b/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs index b96d630e45c..a9df9068bb0 100644 --- a/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs +++ b/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs @@ -119,5 +119,31 @@ public void ValueCount() termBucket.Should().NotBeNull(); termBucket.Value.Should().HaveValue(); } + + [Test] + public void TopHits() + { + var results = this.Client.Search(s => s + .Size(0) + .Aggregations(a => a + .Terms("top-countries", t => t + .Field(p => p.Country) + .Size(3) + .Aggregations(aa => aa + .TopHits("top-country-hits", th => th + .Sort(sort => sort + .OnField(p => p.StartedOn) + .Order(SortOrder.Descending) + ) + .Source(src => src + .Include(p => p.Name) + ) + .Size(1) + ) + ) + ) + ) + ); + } } } \ No newline at end of file From 31e6466fc084748f3e76e48ba964225b3dd3d080 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 11 Sep 2014 13:18:43 +0200 Subject: [PATCH 2/3] added support for Top Hits aggregation --- .../Domain/Aggregations/AggregationsHelper.cs | 5 +++ src/Nest/Domain/Aggregations/TopHitsMetric.cs | 41 +++++++++++++++++++ src/Nest/Domain/Hit/Hit.cs | 4 +- src/Nest/Nest.csproj | 1 + .../Aggregations/AggregationConverter.cs | 33 +++++++++++---- .../Aggregations/MetricAggregationTests.cs | 16 ++++++++ 6 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 src/Nest/Domain/Aggregations/TopHitsMetric.cs diff --git a/src/Nest/Domain/Aggregations/AggregationsHelper.cs b/src/Nest/Domain/Aggregations/AggregationsHelper.cs index 5c3a2b681bf..5953d5ce0b4 100644 --- a/src/Nest/Domain/Aggregations/AggregationsHelper.cs +++ b/src/Nest/Domain/Aggregations/AggregationsHelper.cs @@ -81,6 +81,11 @@ public PercentilesMetric PercentilesRank(string key) return this.TryGet(key); } + public TopHitsMetric TopHitsMetric(string key) + { + return this.TryGet(key); + } + public SingleBucket Global(string key) { return this.TryGet(key); diff --git a/src/Nest/Domain/Aggregations/TopHitsMetric.cs b/src/Nest/Domain/Aggregations/TopHitsMetric.cs new file mode 100644 index 00000000000..67b37a3e96c --- /dev/null +++ b/src/Nest/Domain/Aggregations/TopHitsMetric.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Nest +{ + public class TopHitsMetric : IMetricAggregation + { + private IEnumerable _hits; + + public TopHitsMetric() + { + } + + internal TopHitsMetric(IEnumerable hits) + { + _hits = hits; + } + + public long Total { get; set; } + public double? MaxScore { get; set; } + + public IEnumerable> Hits(JsonSerializer serializer = null) where T : class + { + if (serializer != null) + return _hits.Select(h => h.ToObject>(serializer)); + return _hits.Select(h => h.ToObject>()); + } + + public IEnumerable Documents(JsonSerializer serializer = null) where T : class + { + return this.Hits(serializer).Select(h => h.Source); + } + + + } + +} diff --git a/src/Nest/Domain/Hit/Hit.cs b/src/Nest/Domain/Hit/Hit.cs index 75fa951a1a5..5d6db02d92f 100644 --- a/src/Nest/Domain/Hit/Hit.cs +++ b/src/Nest/Domain/Hit/Hit.cs @@ -12,7 +12,7 @@ public interface IHit where T : class IFieldSelection Fields { get; } T Source { get; } string Index { get; } - double Score { get; } + double? Score { get; } string Type { get; } string Version { get; } string Id { get; } @@ -36,7 +36,7 @@ public class Hit : IHit [JsonProperty(PropertyName = "_index")] public string Index { get; internal set; } [JsonProperty(PropertyName = "_score")] - public double Score { get; internal set; } + public double? Score { get; internal set; } [JsonProperty(PropertyName = "_type")] public string Type { get; internal set; } [JsonProperty(PropertyName = "_version")] diff --git a/src/Nest/Nest.csproj b/src/Nest/Nest.csproj index d2ba885a7ef..f98e76ffb13 100644 --- a/src/Nest/Nest.csproj +++ b/src/Nest/Nest.csproj @@ -106,6 +106,7 @@ + diff --git a/src/Nest/Resolvers/Converters/Aggregations/AggregationConverter.cs b/src/Nest/Resolvers/Converters/Aggregations/AggregationConverter.cs index f9b38f4bcad..3732887aafd 100644 --- a/src/Nest/Resolvers/Converters/Aggregations/AggregationConverter.cs +++ b/src/Nest/Resolvers/Converters/Aggregations/AggregationConverter.cs @@ -59,12 +59,27 @@ private IAggregation ReadAggregation(JsonReader reader, JsonSerializer serialize return GetSingleBucketAggregation(reader, serializer); case "bounds": return GetGeoBoundsMetricAggregation(reader, serializer); + case "hits": + return GetHitsAggregation(reader, serializer); default: return null; } } + private IAggregation GetHitsAggregation(JsonReader reader, JsonSerializer serializer) + { + reader.Read(); + var o = JObject.Load(reader); + if (o == null) + return null; + + var total = o["total"].ToObject(); + var maxScore = o["max_score"].ToObject(); + var hits = o["hits"].Children().OfType().Select(s=>s); + return new TopHitsMetric(hits) { Total = total, MaxScore = maxScore }; + } + private IAggregation GetGeoBoundsMetricAggregation(JsonReader reader, JsonSerializer serializer) { reader.Read(); @@ -369,15 +384,15 @@ public IAggregation GetRangeAggregation(JsonReader reader, JsonSerializer serial break; } } - var bucket = new RangeItem - { - Key = key, - From = fromDouble, - To = toDouble, - DocCount = docCount.GetValueOrDefault(), - FromAsString = fromAsString, - ToAsString = toAsString - }; + var bucket = new RangeItem + { + Key = key, + From = fromDouble, + To = toDouble, + DocCount = docCount.GetValueOrDefault(), + FromAsString = fromAsString, + ToAsString = toAsString + }; bucket.Aggregations = this.GetNestedAggregations(reader, serializer); return bucket; diff --git a/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs b/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs index 0fc598d7f77..2aa999ed99b 100644 --- a/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs +++ b/src/Tests/Nest.Tests.Integration/Aggregations/MetricAggregationTests.cs @@ -1,4 +1,5 @@ using System.Linq; +using Elasticsearch.Net; using FluentAssertions; using Nest.Tests.MockData.Domain; using NUnit.Framework; @@ -187,6 +188,21 @@ public void TopHits() ) ) ); + + results.IsValid.Should().BeTrue(); + + var topCountries = results.Aggs.Terms("top-countries").Items; + foreach(var topCountry in topCountries) + { + var topHits = topCountry.TopHitsMetric("top-country-hits"); + topHits.Should().NotBeNull(); + topHits.Total.Should().BeGreaterThan(0); + var hits = topHits.Hits(); + hits.Should().NotBeEmpty().And.NotContain(h=> h.Id.IsNullOrEmpty() || h.Index.IsNullOrEmpty()); + topHits.Documents().Should().NotBeEmpty(); + + } + } } } \ No newline at end of file From bc2f99dc873f2785ec099508f3170fc90ace3fb7 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 15 Sep 2014 13:47:58 +0200 Subject: [PATCH 3/3] made Score backward compatible --- src/Nest/Domain/Hit/Hit.cs | 8 ++++++-- src/Nest/Resolvers/Converters/ConcreteTypeConverter.cs | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Nest/Domain/Hit/Hit.cs b/src/Nest/Domain/Hit/Hit.cs index 5d6db02d92f..be4a33287a1 100644 --- a/src/Nest/Domain/Hit/Hit.cs +++ b/src/Nest/Domain/Hit/Hit.cs @@ -12,7 +12,7 @@ public interface IHit where T : class IFieldSelection Fields { get; } T Source { get; } string Index { get; } - double? Score { get; } + double Score { get; } string Type { get; } string Version { get; } string Id { get; } @@ -35,8 +35,12 @@ public class Hit : IHit public T Source { get; internal set; } [JsonProperty(PropertyName = "_index")] public string Index { get; internal set; } + + //TODO in NEST 2.0 make the property itself double? [JsonProperty(PropertyName = "_score")] - public double? Score { get; internal set; } + internal double? _score { get; set; } + public double Score { get { return _score.GetValueOrDefault(0); } } + [JsonProperty(PropertyName = "_type")] public string Type { get; internal set; } [JsonProperty(PropertyName = "_version")] diff --git a/src/Nest/Resolvers/Converters/ConcreteTypeConverter.cs b/src/Nest/Resolvers/Converters/ConcreteTypeConverter.cs index 35dd1c872f6..e0636638893 100644 --- a/src/Nest/Resolvers/Converters/ConcreteTypeConverter.cs +++ b/src/Nest/Resolvers/Converters/ConcreteTypeConverter.cs @@ -175,7 +175,7 @@ internal static Type GetConcreteTypeUsingSelector( hitDynamic.Fields = sel; hitDynamic.Source = d._source; hitDynamic.Index = d._index; - hitDynamic.Score = (d._score is double) ? d._score : default(double); + hitDynamic._score = (d._score is double) ? d._score : default(double); hitDynamic.Type = d._type; hitDynamic.Version = d._version; hitDynamic.Id = d._id;