Initial Json.NET serialiser

Author: stevejgordon

Elasticsearch.sln

@@ -56,6 +56,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "scripts", "build\scripts\sc
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests.ClusterLauncher", "tests\Tests.ClusterLauncher\Tests.ClusterLauncher.csproj", "{F6162603-D134-4121-8106-2BA4DAD7350B}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Clients.Elasticsearch.JsonNetSerializer", "src\Elastic.Clients.Elasticsearch.JsonNetSerializer\Elastic.Clients.Elasticsearch.JsonNetSerializer.csproj", "{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -222,6 +224,18 @@ Global
 		{F6162603-D134-4121-8106-2BA4DAD7350B}.Release|x64.Build.0 = Release|Any CPU
 		{F6162603-D134-4121-8106-2BA4DAD7350B}.Release|x86.ActiveCfg = Release|Any CPU
 		{F6162603-D134-4121-8106-2BA4DAD7350B}.Release|x86.Build.0 = Release|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Debug|x64.Build.0 = Debug|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Debug|x86.Build.0 = Debug|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Release|x64.ActiveCfg = Release|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Release|x64.Build.0 = Release|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Release|x86.ActiveCfg = Release|Any CPU
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -240,6 +254,7 @@ Global
 		{11362CEE-B4B3-4EFE-A9A1-A6CDEEFCEA10} = {D455EC79-E1E0-4509-B297-0DA3AED8DFF7}
 		{68D1BFDC-F447-4D2C-AF81-537807636610} = {1FE49D14-216A-41EE-A177-E42BFF53E0DC}
 		{F6162603-D134-4121-8106-2BA4DAD7350B} = {362B2776-4B29-46AB-B237-56776B5372B6}
+		{8C9275D9-29CE-4A20-8FD5-6B26C6CAAADB} = {D455EC79-E1E0-4509-B297-0DA3AED8DFF7}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CE74F821-B001-4C69-A58D-CF81F8B0B632}

build/scripts/Commandline.fs

@@ -71,7 +71,7 @@ Execution hints can be provided anywhere on the command line
     let private (|IsProject|_|) (candidate:string) =
         let c = candidate.ToLowerInvariant()
         match c with
-        | "nest" | "elasticsearch.net" | "nest.jsonnetserializer" -> Some c
+        | "nest" | "elasticsearch.net" | "elastic.clients.elasticsearch.jsonnetserializer" -> Some c
         | _ -> None     
 
     let private (|IsFormat|_|) (candidate:string) =

build/scripts/Testing.fs

@@ -62,7 +62,7 @@ module Tests =
     let RunReleaseUnitTests version args =
         //xUnit always does its own build, this env var is picked up by Tests.csproj
         //if its set it will include the local package source (build/output/)
-        //and references NEST and NEST.JsonNetSerializer by the current version
+        //and references NEST and Elastic.Clients.Elasticsearch.JsonNetSerializer by the current version
         //this works by not including the local package cache (nay source) 
         //in the project file via:
         //<RestoreSources></RestoreSources>

src/Elastic.Clients.Elasticsearch.JsonNetSerializer/ConnectionSettingsAwareContractResolver.cs

@@ -0,0 +1,89 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using Elastic.Clients.Elasticsearch;
+using Elastic.Clients.Elasticsearch.QueryDsl;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Serialization;
+using JsonProperty = Newtonsoft.Json.Serialization.JsonProperty;
+
+namespace Elastic.Clients.JsonNetSerializer
+{
+	public class ConnectionSettingsAwareContractResolver : DefaultContractResolver
+	{
+		public ConnectionSettingsAwareContractResolver(IElasticsearchClientSettings connectionSettings) =>
+			ConnectionSettings = connectionSettings ?? throw new ArgumentNullException(nameof(connectionSettings));
+
+		protected IElasticsearchClientSettings ConnectionSettings { get; }
+
+		protected override string ResolvePropertyName(string fieldName) =>
+			ConnectionSettings.DefaultFieldNameInferrer != null
+				? ConnectionSettings.DefaultFieldNameInferrer(fieldName)
+				: base.ResolvePropertyName(fieldName);
+
+		protected override JsonContract CreateContract(Type objectType)
+		{
+			var contract = base.CreateContract(objectType);
+			if (objectType.IsEnum && objectType.GetCustomAttribute<StringEnumAttribute>() != null)
+				contract.Converter = new StringEnumConverter();
+
+			return contract;
+		}
+
+		protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization)
+		{
+			var property = base.CreateProperty(member, memberSerialization);
+			ApplyShouldSerializer(property);
+			ApplyPropertyOverrides(member, property);
+			return property;
+		}
+
+		/// <summary> Renames/Ignores a property based on the connection settings mapping or custom attributes for the property </summary>
+		private void ApplyPropertyOverrides(MemberInfo member, JsonProperty property)
+		{
+			if (!ConnectionSettings.PropertyMappings.TryGetValue(member, out var propertyMapping))
+				propertyMapping = ElasticsearchPropertyAttributeBase.From(member);
+
+			var serializerMapping = ConnectionSettings.PropertyMappingProvider?.CreatePropertyMapping(member);
+
+			var nameOverride = propertyMapping?.Name ?? serializerMapping?.Name;
+			if (!string.IsNullOrWhiteSpace(nameOverride)) property.PropertyName = nameOverride;
+
+			var overrideIgnore = propertyMapping?.Ignore ?? serializerMapping?.Ignore;
+			if (overrideIgnore.HasValue)
+				property.Ignored = overrideIgnore.Value;
+		}
+
+		private static void ApplyShouldSerializer(JsonProperty property)
+		{
+			if (property.PropertyType == typeof(QueryContainer))
+				property.ShouldSerialize = o => ShouldSerializeQueryContainer(o, property);
+			else if (property.PropertyType == typeof(IEnumerable<QueryContainer>))
+				property.ShouldSerialize = o => ShouldSerializeQueryContainers(o, property);
+		}
+
+		private static bool ShouldSerializeQueryContainer(object o, JsonProperty prop)
+		{
+			if (o == null) return false;
+			if (prop.ValueProvider.GetValue(o) is not QueryContainer q) return false;
+			//return q.IsWritable;
+			return true;
+		}
+
+		private static bool ShouldSerializeQueryContainers(object o, JsonProperty prop)
+		{
+			if (o == null) return false;
+			if (prop.ValueProvider.GetValue(o) is not IEnumerable<QueryContainer> q) return false;
+
+			var queryContainers = q as QueryContainer[] ?? q.ToArray();
+			//return queryContainers.Any(qq => qq != null && ((QueryContainer)qq).IsWritable);
+			return queryContainers.Any(qq => qq != null);
+		}
+	}
+}

src/Elastic.Clients.Elasticsearch.JsonNetSerializer/ConnectionSettingsAwareSerializerBase.Customization.cs

@@ -0,0 +1,75 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Elastic.Transport;
+using Newtonsoft.Json;
+
+namespace Elastic.Clients.JsonNetSerializer
+{
+	public abstract partial class ConnectionSettingsAwareSerializerBase : SerializerBase
+	{
+		// Default buffer size of StreamWriter, which is private :(
+		internal const int DefaultBufferSize = 1024;
+
+		private static readonly Task CompletedTask = Task.CompletedTask;
+
+		internal static readonly Encoding ExpectedEncoding = new UTF8Encoding(false);
+		private readonly JsonSerializer _collapsedSerializer;
+
+		private readonly JsonSerializer _serializer;
+		protected virtual int BufferSize => DefaultBufferSize;
+
+		public override T Deserialize<T>(Stream stream)
+		{
+			using var streamReader = new StreamReader(stream);
+			using var jsonTextReader = new JsonTextReader(streamReader);
+			return _serializer.Deserialize<T>(jsonTextReader);
+		}
+
+		public override object Deserialize(Type type, Stream stream)
+		{
+			using var streamReader = new StreamReader(stream);
+			using var jsonTextReader = new JsonTextReader(streamReader);
+			return _serializer.Deserialize(jsonTextReader, type);
+		}
+
+		public override async Task<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default)
+		{
+			using var streamReader = new StreamReader(stream);
+			using var jsonTextReader = new JsonTextReader(streamReader);
+			var token = await jsonTextReader.ReadTokenWithDateParseHandlingNoneAsync(cancellationToken).ConfigureAwait(false);
+			return token.ToObject<T>(_serializer);
+		}
+
+		public override async Task<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default(CancellationToken))
+		{
+			using var streamReader = new StreamReader(stream);
+			using var jsonTextReader = new JsonTextReader(streamReader);
+			var token = await jsonTextReader.ReadTokenWithDateParseHandlingNoneAsync(cancellationToken).ConfigureAwait(false);
+			return token.ToObject(type, _serializer);
+		}
+
+		public override void Serialize<T>(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None)
+		{
+			using var writer = new StreamWriter(stream, ExpectedEncoding, BufferSize, true);
+			using var jsonWriter = new JsonTextWriter(writer);
+			var serializer = formatting == SerializationFormatting.Indented ? _serializer : _collapsedSerializer;
+			serializer.Serialize(jsonWriter, data);
+		}
+
+		public override Task SerializeAsync<T>(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None,
+			CancellationToken cancellationToken = default)
+		{
+			//This makes no sense now but we need the async method on the interface in 6.x so we can start swapping this out
+			//for an implementation that does make sense without having to wait for 7.x
+			Serialize(data, stream, formatting);
+			return CompletedTask;
+		}
+	}
+}

src/Elastic.Clients.Elasticsearch.JsonNetSerializer/ConnectionSettingsAwareSerializerBase.PropertyMappingProvider.cs

@@ -0,0 +1,40 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Concurrent;
+using System.Reflection;
+using System.Runtime.Serialization;
+using Elastic.Clients.Elasticsearch;
+using Newtonsoft.Json;
+
+namespace Elastic.Clients.JsonNetSerializer
+{
+	public abstract partial class ConnectionSettingsAwareSerializerBase : IPropertyMappingProvider
+	{
+		protected readonly ConcurrentDictionary<string, PropertyMapping> Properties = new();
+
+		public PropertyMapping CreatePropertyMapping(MemberInfo memberInfo)
+		{
+			var memberInfoString = $"{memberInfo.DeclaringType?.FullName}.{memberInfo.Name}";
+			if (Properties.TryGetValue(memberInfoString, out var mapping)) return mapping;
+			mapping = FromAttributes(memberInfo);
+
+			Properties.TryAdd(memberInfoString, mapping);
+			return mapping;
+		}
+
+		private static PropertyMapping FromAttributes(MemberInfo memberInfo)
+		{
+			var jsonProperty = memberInfo.GetCustomAttribute<JsonPropertyAttribute>(true);
+			var dataMemberProperty = memberInfo.GetCustomAttribute<DataMemberAttribute>(true);
+			var propertyName = memberInfo.GetCustomAttribute<PropertyNameAttribute>(true);
+			var ignore = memberInfo.GetCustomAttribute<IgnoreAttribute>(true);
+			var jsonIgnore = memberInfo.GetCustomAttribute<JsonIgnoreAttribute>(true);
+			if (jsonProperty == null && ignore == null && propertyName == null && dataMemberProperty == null && jsonIgnore == null) return null;
+
+			return new PropertyMapping
+				{ Name = propertyName?.Name ?? jsonProperty?.PropertyName ?? dataMemberProperty?.Name, Ignore = ignore != null || jsonIgnore != null };
+		}
+	}
+}

src/Elastic.Clients.Elasticsearch.JsonNetSerializer/ConnectionSettingsAwareSerializerBase.Serializer.cs

@@ -0,0 +1,79 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Elastic.Clients.JsonNetSerializer.Converters;
+using Newtonsoft.Json;
+using Elastic.Transport;
+using Elastic.Clients.Elasticsearch;
+
+namespace Elastic.Clients.JsonNetSerializer
+{
+	public abstract partial class ConnectionSettingsAwareSerializerBase
+	{
+		protected ConnectionSettingsAwareSerializerBase(SerializerBase builtinSerializer, IElasticsearchClientSettings connectionSettings)
+			: this(builtinSerializer, connectionSettings, null, null, null) { }
+
+		internal ConnectionSettingsAwareSerializerBase(
+			SerializerBase builtinSerializer,
+			IElasticsearchClientSettings connectionSettings,
+			Func<JsonSerializerSettings> jsonSerializerSettingsFactory,
+			Action<ConnectionSettingsAwareContractResolver> modifyContractResolver,
+			IEnumerable<JsonConverter> contractJsonConverters
+		)
+		{
+			JsonSerializerSettingsFactory = jsonSerializerSettingsFactory;
+			ModifyContractResolverCallback = modifyContractResolver;
+			ContractJsonConverters = contractJsonConverters ?? Enumerable.Empty<JsonConverter>();
+
+			ConnectionSettings = connectionSettings;
+			BuiltinSerializer = builtinSerializer;
+			Converters = new List<JsonConverter>
+			{
+				new HandleNestTypesOnSourceJsonConverter(BuiltinSerializer, connectionSettings.MemoryStreamFactory),
+				new TimeSpanToStringConverter()
+			};
+			_serializer = CreateSerializer(SerializationFormatting.Indented);
+			_collapsedSerializer = CreateSerializer(SerializationFormatting.None);
+		}
+
+		protected SerializerBase BuiltinSerializer { get; }
+
+		protected IElasticsearchClientSettings ConnectionSettings { get; }
+		protected IEnumerable<JsonConverter> ContractJsonConverters { get; }
+		protected Func<JsonSerializerSettings> JsonSerializerSettingsFactory { get; }
+		protected Action<ConnectionSettingsAwareContractResolver> ModifyContractResolverCallback { get; }
+
+		private List<JsonConverter> Converters { get; }
+
+		private JsonSerializer CreateSerializer(SerializationFormatting formatting)
+		{
+			var s = CreateJsonSerializerSettings() ?? new JsonSerializerSettings();
+			var converters = CreateJsonConverters() ?? Enumerable.Empty<JsonConverter>();
+			var contract = CreateContractResolver();
+			s.Formatting = formatting == SerializationFormatting.Indented ? Formatting.Indented : Formatting.None;
+			s.ContractResolver = contract;
+			foreach (var converter in converters.Concat(Converters))
+				s.Converters.Add(converter);
+
+			return JsonSerializer.Create(s);
+		}
+
+		protected virtual ConnectionSettingsAwareContractResolver CreateContractResolver()
+		{
+			var contract = new ConnectionSettingsAwareContractResolver(ConnectionSettings);
+			ModifyContractResolver(contract);
+			return contract;
+		}
+
+		protected virtual JsonSerializerSettings CreateJsonSerializerSettings() => JsonSerializerSettingsFactory?.Invoke();
+
+		protected virtual IEnumerable<JsonConverter> CreateJsonConverters() => ContractJsonConverters;
+
+		protected virtual void ModifyContractResolver(ConnectionSettingsAwareContractResolver resolver) =>
+			ModifyContractResolverCallback?.Invoke(resolver);
+	}
+}