From 562544c411c0b39b7bbad267d3f39f8b988a4167 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 11 Apr 2019 13:28:58 +1000 Subject: [PATCH] Serialize fractional numbers with trailing decimal point and value This commit updates - the DoubleToStringConverter from utf8json to always emit a trailing decimal point and trailing decimal value when serializing floating point numbers to JSON. - the DecimalFormatter from utf8json to always emit a trailing decimal point and trailing decimal value when serializing floating point numbers to JSON. Update unit tests to reflect this change Fixes #3657 --- .../StandardClassLibraryFormatters.cs | 6 +- .../DoubleToStringConverter.cs | 3 +- .../StringToDoubleConverter.cs | 1 + .../Utf8Json/JsonSerializer.cs | 24 ++++---- .../Suggesters/TermSuggester/TermSuggester.cs | 1 - .../Serialization/SerializationTester.cs | 2 +- .../Serialization/Utf8JsonDecimalConverter.cs | 34 ----------- .../MatrixStatsAggregationUsageTests.cs | 4 +- .../Serialization/FractionalNumbers.cs | 60 +++++++++++++++++++ .../Compound/Bool/BoolQueryUsageTests.cs | 2 +- .../FunctionScoreQueryUsageTests.cs | 12 ++-- .../GeoBoundingBoxQueryUsageTests.cs | 8 +-- .../Distance/GeoDistanceQueryUsageTests.cs | 4 +- .../Geo/Polygon/GeoPolygonQueryUsageTests.cs | 4 +- 14 files changed, 95 insertions(+), 70 deletions(-) delete mode 100644 src/Tests/Tests.Core/Serialization/Utf8JsonDecimalConverter.cs create mode 100644 src/Tests/Tests/CodeStandards/Serialization/FractionalNumbers.cs diff --git a/src/Elasticsearch.Net/Utf8Json/Formatters/StandardClassLibraryFormatters.cs b/src/Elasticsearch.Net/Utf8Json/Formatters/StandardClassLibraryFormatters.cs index f07ae92fb47..a93ae3c83ae 100644 --- a/src/Elasticsearch.Net/Utf8Json/Formatters/StandardClassLibraryFormatters.cs +++ b/src/Elasticsearch.Net/Utf8Json/Formatters/StandardClassLibraryFormatters.cs @@ -307,14 +307,16 @@ public DecimalFormatter(bool serializeAsString) public void Serialize(ref JsonWriter writer, decimal value, IJsonFormatterResolver formatterResolver) { + // always include decimal point and at least one decimal place + var s = value.ToString("0.0###########################", CultureInfo.InvariantCulture); if (serializeAsString) { - writer.WriteString(value.ToString(CultureInfo.InvariantCulture)); + writer.WriteString(s); } else { // write as number format. - writer.WriteRaw(StringEncoding.UTF8.GetBytes(value.ToString(CultureInfo.InvariantCulture))); + writer.WriteRaw(StringEncoding.UTF8.GetBytes(s)); } } diff --git a/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/DoubleToStringConverter.cs b/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/DoubleToStringConverter.cs index a162b203396..68806f3121b 100644 --- a/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/DoubleToStringConverter.cs +++ b/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/DoubleToStringConverter.cs @@ -185,6 +185,7 @@ enum DtoaMode // PRECISION } + [Flags] enum Flags { NO_FLAGS = 0, @@ -214,7 +215,7 @@ enum Flags //const int max_leading_padding_zeroes_in_precision_mode_; //const int max_trailing_padding_zeroes_in_precision_mode_; - static readonly Flags flags_ = Flags.UNIQUE_ZERO | Flags.EMIT_POSITIVE_EXPONENT_SIGN; + static readonly Flags flags_ = Flags.UNIQUE_ZERO | Flags.EMIT_POSITIVE_EXPONENT_SIGN | Flags.EMIT_TRAILING_DECIMAL_POINT | Flags.EMIT_TRAILING_ZERO_AFTER_POINT; static readonly char exponent_character_ = 'E'; static readonly int decimal_in_shortest_low_ = -4; // C# ToString("G") static readonly int decimal_in_shortest_high_ = 15;// C# ToString("G") diff --git a/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/StringToDoubleConverter.cs b/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/StringToDoubleConverter.cs index 489560f5890..72bbcdb8d61 100644 --- a/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/StringToDoubleConverter.cs +++ b/src/Elasticsearch.Net/Utf8Json/Internal/DoubleConversion/StringToDoubleConverter.cs @@ -160,6 +160,7 @@ public static float ToSingle(byte[] buffer, int offset, out int readCount) // port internal static partial class StringToDoubleConverter { + [Flags] enum Flags { NO_FLAGS = 0, diff --git a/src/Elasticsearch.Net/Utf8Json/JsonSerializer.cs b/src/Elasticsearch.Net/Utf8Json/JsonSerializer.cs index 95e64d2605a..d929d2a007f 100644 --- a/src/Elasticsearch.Net/Utf8Json/JsonSerializer.cs +++ b/src/Elasticsearch.Net/Utf8Json/JsonSerializer.cs @@ -285,8 +285,7 @@ public static T Deserialize(Stream stream, IJsonFormatterResolver resolver) var ms = stream as MemoryStream; if (ms != null) { - ArraySegment buf2; - if (ms.TryGetBuffer(out buf2)) + if (ms.TryGetBuffer(out var buf2)) { // when token is number, can not use from pool(can not find end line). var token = new JsonReader(buf2.Array, buf2.Offset).GetCurrentJsonToken(); @@ -367,19 +366,16 @@ public static async System.Threading.Tasks.Task DeserializeAsync(Stream st #endif - public static string PrettyPrint(byte[] json) - { - return PrettyPrint(json, 0); - } + public static string PrettyPrint(byte[] json) => PrettyPrint(json, 0); - public static string PrettyPrint(byte[] json, int offset) + public static string PrettyPrint(byte[] json, int offset) { var reader = new JsonReader(json, offset); var buffer = MemoryPool.Rent(); try { var writer = new JsonWriter(buffer); - WritePrittyPrint(ref reader, ref writer, 0); + WritePrettyPrint(ref reader, ref writer, 0); return writer.ToString(); } finally @@ -396,7 +392,7 @@ public static string PrettyPrint(string json) try { var writer = new JsonWriter(buffer); - WritePrittyPrint(ref reader, ref writer, 0); + WritePrettyPrint(ref reader, ref writer, 0); return writer.ToString(); } finally @@ -418,7 +414,7 @@ public static byte[] PrettyPrintByteArray(byte[] json, int offset) try { var writer = new JsonWriter(buffer); - WritePrittyPrint(ref reader, ref writer, 0); + WritePrettyPrint(ref reader, ref writer, 0); return writer.ToUtf8ByteArray(); } finally @@ -434,7 +430,7 @@ public static byte[] PrettyPrintByteArray(string json) try { var writer = new JsonWriter(buffer); - WritePrittyPrint(ref reader, ref writer, 0); + WritePrettyPrint(ref reader, ref writer, 0); return writer.ToUtf8ByteArray(); } finally @@ -446,7 +442,7 @@ public static byte[] PrettyPrintByteArray(string json) static readonly byte[][] indent = Enumerable.Range(0, 100).Select(x => Encoding.UTF8.GetBytes(new string(' ', x * 2))).ToArray(); static readonly byte[] newLine = Encoding.UTF8.GetBytes(Environment.NewLine); - static void WritePrittyPrint(ref JsonReader reader, ref JsonWriter writer, int depth) + static void WritePrettyPrint(ref JsonReader reader, ref JsonWriter writer, int depth) { var token = reader.GetCurrentJsonToken(); switch (token) @@ -466,7 +462,7 @@ static void WritePrittyPrint(ref JsonReader reader, ref JsonWriter writer, int d writer.WriteRaw(indent[depth + 1]); writer.WritePropertyName(reader.ReadPropertyName()); writer.WriteRaw((byte)' '); - WritePrittyPrint(ref reader, ref writer, depth + 1); + WritePrettyPrint(ref reader, ref writer, depth + 1); } writer.WriteRaw(newLine); writer.WriteRaw(indent[depth]); @@ -486,7 +482,7 @@ static void WritePrittyPrint(ref JsonReader reader, ref JsonWriter writer, int d writer.WriteRaw(newLine); } writer.WriteRaw(indent[depth + 1]); - WritePrittyPrint(ref reader, ref writer, depth + 1); + WritePrettyPrint(ref reader, ref writer, depth + 1); } writer.WriteRaw(newLine); writer.WriteRaw(indent[depth]); diff --git a/src/Nest/Search/Suggesters/TermSuggester/TermSuggester.cs b/src/Nest/Search/Suggesters/TermSuggester/TermSuggester.cs index 86af1d06c5b..edc03c71e5c 100644 --- a/src/Nest/Search/Suggesters/TermSuggester/TermSuggester.cs +++ b/src/Nest/Search/Suggesters/TermSuggester/TermSuggester.cs @@ -38,7 +38,6 @@ public interface ITermSuggester : ISuggester StringDistance? StringDistance { get; set; } [DataMember(Name = "suggest_mode")] - SuggestMode? SuggestMode { get; set; } [IgnoreDataMember] diff --git a/src/Tests/Tests.Core/Serialization/SerializationTester.cs b/src/Tests/Tests.Core/Serialization/SerializationTester.cs index 739aec9464b..91c4a7753c0 100644 --- a/src/Tests/Tests.Core/Serialization/SerializationTester.cs +++ b/src/Tests/Tests.Core/Serialization/SerializationTester.cs @@ -237,7 +237,7 @@ private JsonSerializerSettings ExpectedJsonSerializerSettings(bool preserveNullI ContractResolver = new DefaultContractResolver { NamingStrategy = new DefaultNamingStrategy() }, NullValueHandling = preserveNullInExpected ? NullValueHandling.Include : NullValueHandling.Ignore, //copied here because anonymyzing geocoordinates is too tedious - Converters = new List { new TestGeoCoordinateJsonConverter(), new Utf8JsonDecimalConverter() } + Converters = new List { new TestGeoCoordinateJsonConverter() } }; } } diff --git a/src/Tests/Tests.Core/Serialization/Utf8JsonDecimalConverter.cs b/src/Tests/Tests.Core/Serialization/Utf8JsonDecimalConverter.cs deleted file mode 100644 index f5ad1a2060d..00000000000 --- a/src/Tests/Tests.Core/Serialization/Utf8JsonDecimalConverter.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using Newtonsoft.Json; -using static System.Decimal; - -namespace Tests.Core.Serialization -{ - /// - /// Serializes decimal, float and double with no decimal places - /// when the contained value is a whole number - /// - /// - /// JSON.Net always appends .0 for floating points and decimal, so remove - /// when serializing - /// - public class Utf8JsonDecimalConverter : JsonConverter - { - public override bool CanRead => false; - - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) => - throw new NotSupportedException(); - - public override bool CanConvert(Type objectType) => - objectType == typeof(decimal) || objectType == typeof(float) || objectType == typeof(double) || - objectType == typeof(decimal?) || objectType == typeof(float?) || objectType == typeof(double?); - - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) - { - var stringValue = JsonConvert.ToString(value); - writer.WriteRawValue(stringValue.EndsWith(".0") - ? stringValue.Substring(0, stringValue.IndexOf(".")) - : stringValue); - } - } -} diff --git a/src/Tests/Tests/Aggregations/Matrix/MatrixStats/MatrixStatsAggregationUsageTests.cs b/src/Tests/Tests/Aggregations/Matrix/MatrixStats/MatrixStatsAggregationUsageTests.cs index e6b7a230d4e..63e2676f5c1 100644 --- a/src/Tests/Tests/Aggregations/Matrix/MatrixStats/MatrixStatsAggregationUsageTests.cs +++ b/src/Tests/Tests/Aggregations/Matrix/MatrixStats/MatrixStatsAggregationUsageTests.cs @@ -28,8 +28,8 @@ public MatrixStatsAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) fields = new[] { "numberOfCommits", "numberOfContributors" }, missing = new { - numberOfCommits = 0, - numberOfContributors = 1 + numberOfCommits = 0.0, + numberOfContributors = 1.0 }, mode = "median" } diff --git a/src/Tests/Tests/CodeStandards/Serialization/FractionalNumbers.cs b/src/Tests/Tests/CodeStandards/Serialization/FractionalNumbers.cs new file mode 100644 index 00000000000..deffed87fd4 --- /dev/null +++ b/src/Tests/Tests/CodeStandards/Serialization/FractionalNumbers.cs @@ -0,0 +1,60 @@ +using System; +using Elastic.Xunit.XunitPlumbing; +using Elasticsearch.Net; +using FluentAssertions; +using Nest; + +namespace Tests.CodeStandards.Serialization +{ + public class FractionalNumbers + { + private readonly IElasticsearchSerializer _serializer; + + public FractionalNumbers() + { + var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + var settings = new ConnectionSettings(pool, new InMemoryConnection()); + var client = new ElasticClient(settings); + _serializer = client.RequestResponseSerializer; + } + + [U] + public void SerializeDouble() + { + var poco = new + { + Whole = 1d, + Fractional = 1.1d + }; + + var serialized = _serializer.SerializeToString(poco); + serialized.Should().Be("{\"whole\":1.0,\"fractional\":1.1}"); + } + + [U] + public void SerializeFloat() + { + var poco = new + { + Whole = 1f, + Fractional = 1.1f + }; + + var serialized = _serializer.SerializeToString(poco); + serialized.Should().Be("{\"whole\":1.0,\"fractional\":1.1}"); + } + + [U] + public void SerializeDecimal() + { + var poco = new + { + Whole = 1m, + Fractional = 1.1m + }; + + var serialized = _serializer.SerializeToString(poco); + serialized.Should().Be("{\"whole\":1.0,\"fractional\":1.1}"); + } + } +} diff --git a/src/Tests/Tests/QueryDsl/Compound/Bool/BoolQueryUsageTests.cs b/src/Tests/Tests/QueryDsl/Compound/Bool/BoolQueryUsageTests.cs index b4da142f6ee..b287d7b5fc1 100644 --- a/src/Tests/Tests/QueryDsl/Compound/Bool/BoolQueryUsageTests.cs +++ b/src/Tests/Tests/QueryDsl/Compound/Bool/BoolQueryUsageTests.cs @@ -127,7 +127,7 @@ public BoolQueryUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base( new { match_all = new { } } }, minimum_should_match = 1, - boost = 2, + boost = 2.0, } }; diff --git a/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs b/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs index 7fcd3d77d63..dfa7a0f0ab4 100644 --- a/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs +++ b/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs @@ -71,7 +71,7 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba { numberOfCommits = new { - origin = 1, + origin = 1.0, scale = 0.1, decay = 0.5 } @@ -98,8 +98,8 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba { origin = new { - lat = 70, - lon = -70 + lat = 70.0, + lon = -70.0 }, scale = "1mi" }, @@ -118,7 +118,7 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba }, new { random_score = new { seed = 1337, field = "_seq_no" } }, new { random_score = new { seed = "randomstring", field = "_seq_no" } }, - new { weight = 1 }, + new { weight = 1.0 }, new { script_score = new @@ -130,8 +130,8 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba } } }, - max_boost = 20, - min_score = 1, + max_boost = 20.0, + min_score = 1.0, query = new { match_all = new { } diff --git a/src/Tests/Tests/QueryDsl/Geo/BoundingBox/GeoBoundingBoxQueryUsageTests.cs b/src/Tests/Tests/QueryDsl/Geo/BoundingBox/GeoBoundingBoxQueryUsageTests.cs index 1afa27190e3..d2d4be98c4d 100644 --- a/src/Tests/Tests/QueryDsl/Geo/BoundingBox/GeoBoundingBoxQueryUsageTests.cs +++ b/src/Tests/Tests/QueryDsl/Geo/BoundingBox/GeoBoundingBoxQueryUsageTests.cs @@ -42,13 +42,13 @@ public GeoBoundingBoxQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : b { top_left = new { - lat = 34, - lon = -34 + lat = 34.0, + lon = -34.0 }, bottom_right = new { - lat = -34, - lon = 34 + lat = -34.0, + lon = 34.0 } } } diff --git a/src/Tests/Tests/QueryDsl/Geo/Distance/GeoDistanceQueryUsageTests.cs b/src/Tests/Tests/QueryDsl/Geo/Distance/GeoDistanceQueryUsageTests.cs index da1b56842f3..af393252c1b 100644 --- a/src/Tests/Tests/QueryDsl/Geo/Distance/GeoDistanceQueryUsageTests.cs +++ b/src/Tests/Tests/QueryDsl/Geo/Distance/GeoDistanceQueryUsageTests.cs @@ -38,8 +38,8 @@ public GeoDistanceQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base validation_method = "ignore_malformed", location = new { - lat = 34, - lon = -34 + lat = 34.0, + lon = -34.0 } } }; diff --git a/src/Tests/Tests/QueryDsl/Geo/Polygon/GeoPolygonQueryUsageTests.cs b/src/Tests/Tests/QueryDsl/Geo/Polygon/GeoPolygonQueryUsageTests.cs index 90aa70b140b..3cd8b5bd005 100644 --- a/src/Tests/Tests/QueryDsl/Geo/Polygon/GeoPolygonQueryUsageTests.cs +++ b/src/Tests/Tests/QueryDsl/Geo/Polygon/GeoPolygonQueryUsageTests.cs @@ -37,8 +37,8 @@ public GeoPolygonQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base( { points = new[] { - new { lat = 45, lon = -45 }, - new { lat = -34, lon = 34 } + new { lat = 45.0, lon = -45.0 }, + new { lat = -34.0, lon = 34.0 } } } }