Skip to content

Fix Extended stats deserialization #4109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 30, 2019
Merged

Fix Extended stats deserialization #4109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0319c1b
Fix issue with extended_stats aggregation on datetime field
benbenwilde Sep 26, 2019
bc2ff70
Also apply the fix for parsing the end of an extended_stats response.
benbenwilde Sep 26, 2019
7c26d94
Merge branch '6.x' of https://github.com/benbenwilde/elasticsearch-ne…
russcam Sep 30, 2019
f0ec574
Assert ExtendedStats deserialization in unit test
russcam Sep 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 27 additions & 17 deletions src/Nest/Aggregations/AggregateJsonConverter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,18 +311,26 @@ private IAggregate GetStatsAggregate(JsonReader reader, JsonSerializer serialize
if (reader.TokenType == JsonToken.EndObject) return new GeoCentroidAggregate { Count = count };

reader.Read();
var min = reader.Value as double?;
var min = reader.TokenType == JsonToken.Integer
? reader.Value as long?
: reader.Value as double?;
reader.Read();
reader.Read();
var max = reader.Value as double?;
var max = reader.TokenType == JsonToken.Integer
? reader.Value as long?
: reader.Value as double?;
reader.Read();
reader.Read();
var average = reader.Value as double?;
var average = reader.TokenType == JsonToken.Integer
? reader.Value as long?
: reader.Value as double?;
reader.Read();
reader.Read();
var sum = reader.Value as double?;
var sum = reader.TokenType == JsonToken.Integer
? reader.Value as long?
: reader.Value as double?;

var statsMetric = new StatsAggregate()
var statsMetric = new StatsAggregate
{
Average = average,
Count = count,
Expand All @@ -336,8 +344,7 @@ private IAggregate GetStatsAggregate(JsonReader reader, JsonSerializer serialize
if (reader.TokenType == JsonToken.EndObject)
return statsMetric;

var propertyName = (string)reader.Value;
while (reader.TokenType != JsonToken.EndObject && propertyName.Contains(Parser.AsStringSuffix))
while (reader.TokenType != JsonToken.EndObject && ((string)reader.Value).EndsWith(Parser.AsStringSuffix))
{
reader.Read();
reader.Read();
Expand All @@ -351,7 +358,7 @@ private IAggregate GetStatsAggregate(JsonReader reader, JsonSerializer serialize

private IAggregate GetExtendedStatsAggregate(StatsAggregate statsMetric, JsonReader reader)
{
var extendedStatsMetric = new ExtendedStatsAggregate()
var extendedStatsMetric = new ExtendedStatsAggregate
{
Average = statsMetric.Average,
Count = statsMetric.Count,
Expand All @@ -361,24 +368,28 @@ private IAggregate GetExtendedStatsAggregate(StatsAggregate statsMetric, JsonRea
};

reader.Read();
extendedStatsMetric.SumOfSquares = reader.Value as double?;
extendedStatsMetric.SumOfSquares = reader.TokenType == JsonToken.Integer
? reader.Value as long?
: reader.Value as double?;
reader.Read();
reader.Read();
extendedStatsMetric.Variance = reader.Value as double?;
extendedStatsMetric.Variance = reader.TokenType == JsonToken.Integer
? reader.Value as long?
: reader.Value as double?;
reader.Read();
reader.Read();
extendedStatsMetric.StdDeviation = reader.Value as double?;
extendedStatsMetric.StdDeviation = reader.TokenType == JsonToken.Integer
? reader.Value as long?
: reader.Value as double?;
reader.Read();

string propertyName;

if (reader.TokenType != JsonToken.EndObject)
{
var bounds = new StandardDeviationBounds();
reader.Read();
reader.Read();

propertyName = (string)reader.Value;
var propertyName = (string)reader.Value;
if (propertyName == Parser.Upper)
{
reader.Read();
Expand All @@ -397,11 +408,10 @@ private IAggregate GetExtendedStatsAggregate(StatsAggregate statsMetric, JsonRea
reader.Read();
}

propertyName = (string)reader.Value;
while (reader.TokenType != JsonToken.EndObject && propertyName.Contains(Parser.AsStringSuffix))
while (reader.TokenType != JsonToken.EndObject && ((string)reader.Value).EndsWith(Parser.AsStringSuffix))
{
// std_deviation_bounds is an object, so we need to skip its properties
if (propertyName.Equals(Parser.StdDeviationBoundsAsString))
if (((string)reader.Value).Equals(Parser.StdDeviationBoundsAsString))
{
reader.Read();
reader.Read();
Expand Down
84 changes: 84 additions & 0 deletions src/Tests/Tests.Reproduce/GitHubIssue4103.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
using System;
using System.Text;
using Elastic.Xunit.XunitPlumbing;
using Elasticsearch.Net;
using FluentAssertions;
using Nest;

namespace Tests.Reproduce
{
public class GithubIssue4013
{
[U]
public void CanDeserializeExtendedStats()
{
var json = @"{
""took"" : 1,
""timed_out"" : false,
""_shards"" : {
""total"" : 2,
""successful"" : 2,
""skipped"" : 0,
""failed"" : 0
},
""hits"" : {
""total"" : 1100,
""max_score"" : 0.0,
""hits"" : [ ]
},
""aggregations"": {
""extended_stats#1"": {
""count"": 3,
""min"": 1569521764937,
""max"": 1569526264937,
""avg"": 1569524464937,
""sum"": 4708573394811,
""min_as_string"": ""2019-09-26T18:16:04.937Z"",
""max_as_string"": ""2019-09-26T19:31:04.937Z"",
""avg_as_string"": ""2019-09-26T19:01:04.937Z"",
""sum_as_string"": ""2119-03-18T09:03:14.811Z"",
""sum_of_squares"": 7.390221138118668e+24,
""variance"": 3779929134421.3335,
""std_deviation"": 1944203.9847766317,
""std_deviation_bounds"": {
""upper"": 1569528353344.9695,
""lower"": 1569520576529.0305
},
""sum_of_squares_as_string"": ""292278994-08-17T07:12:55.807Z"",
""variance_as_string"": ""2089-10-12T04:18:54.421Z"",
""std_deviation_as_string"": ""1970-01-01T00:32:24.203Z"",
""std_deviation_bounds_as_string"": {
""upper"": ""2019-09-26T20:05:53.344Z"",
""lower"": ""2019-09-26T17:56:16.529Z""
}
}
}
}";

var bytes = Encoding.UTF8.GetBytes(json);
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var connectionSettings = new ConnectionSettings(pool, new InMemoryConnection(bytes));
var client = new ElasticClient(connectionSettings);

Action searchResponse = () => client.Search<object>(s => s.AllIndices());

searchResponse.ShouldNotThrow();

var response = client.Search<object>(s => s.AllIndices());

var extendedStats = response.Aggregations.ExtendedStats("1");
extendedStats.Should().NotBeNull();
extendedStats.Count.Should().Be(3);
extendedStats.Min.Should().Be(1569521764937);
extendedStats.Max.Should().Be(1569526264937);
extendedStats.Average.Should().Be(1569524464937);
extendedStats.Sum.Should().Be(4708573394811);
extendedStats.SumOfSquares.Should().Be(7.390221138118668e+24);
extendedStats.Variance.Should().Be(3779929134421.3335);
extendedStats.StdDeviation.Should().Be(1944203.9847766317);
extendedStats.StdDeviationBounds.Should().NotBeNull();
extendedStats.StdDeviationBounds.Upper.Should().Be(1569528353344.9695);
extendedStats.StdDeviationBounds.Lower.Should().Be(1569520576529.0305);
}
}
}