Improved bulk API testing

Author: stevejgordon

Elasticsearch.sln

@@ -56,8 +56,6 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "scripts", "build\scripts\sc
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests.ClusterLauncher", "tests\Tests.ClusterLauncher\Tests.ClusterLauncher.csproj", "{F6162603-D134-4121-8106-2BA4DAD7350B}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elasticsearch.Net.VirtualizedCluster", "src\Elasticsearch.Net.VirtualizedCluster\Elasticsearch.Net.VirtualizedCluster.csproj", "{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -224,18 +222,6 @@ Global
 		{F6162603-D134-4121-8106-2BA4DAD7350B}.Release|x64.Build.0 = Release|Any CPU
 		{F6162603-D134-4121-8106-2BA4DAD7350B}.Release|x86.ActiveCfg = Release|Any CPU
 		{F6162603-D134-4121-8106-2BA4DAD7350B}.Release|x86.Build.0 = Release|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Debug|x64.Build.0 = Debug|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Debug|x86.Build.0 = Debug|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Release|x64.ActiveCfg = Release|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Release|x64.Build.0 = Release|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Release|x86.ActiveCfg = Release|Any CPU
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -254,7 +240,6 @@ Global
 		{11362CEE-B4B3-4EFE-A9A1-A6CDEEFCEA10} = {D455EC79-E1E0-4509-B297-0DA3AED8DFF7}
 		{68D1BFDC-F447-4D2C-AF81-537807636610} = {1FE49D14-216A-41EE-A177-E42BFF53E0DC}
 		{F6162603-D134-4121-8106-2BA4DAD7350B} = {362B2776-4B29-46AB-B237-56776B5372B6}
-		{DFF12151-BB2A-479D-97C6-F4FF6815AA7F} = {D455EC79-E1E0-4509-B297-0DA3AED8DFF7}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CE74F821-B001-4C69-A58D-CF81F8B0B632}

src/Elastic.Clients.Elasticsearch/Common/Infer/Inferrer.cs

@@ -11,6 +11,8 @@
 using System.Text;
 using System.Runtime.Serialization;
 using System.Collections;
+using System.Text.Json.Serialization;
+using System.Text.Json;
 
 namespace Elastic.Clients.Elasticsearch
 {
@@ -228,6 +230,7 @@ private static PropertyInfo GetJoinFieldProperty(Type type)
 		}
 	}
 
+	[JsonConverter(typeof(JoinFieldConverter))]
 	public class JoinField
 	{
 		internal Child ChildOption { get; }
@@ -313,6 +316,23 @@ public Child(RelationName name, Id parent)
 		}
 	}
 
+	internal sealed class JoinFieldConverter : JsonConverter<JoinField>
+	{
+		public override JoinField? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => throw new NotImplementedException();
+		public override void Write(Utf8JsonWriter writer, JoinField value, JsonSerializerOptions options)
+		{
+			switch (value.Tag)
+			{
+				case 0:
+					JsonSerializer.Serialize(writer, value.ParentOption.Name, options);
+					break; 
+			}
+
+			// TODO - Finish this
+		}
+	}
+
+	[JsonConverter(typeof(RelationNameConverter))]
 	public class RelationName : IEquatable<RelationName>, IUrlParameter
 	{
 		private RelationName(string type) => Name = type;
@@ -382,6 +402,26 @@ public bool EqualsMarker(RelationName other)
 
 	}
 
+	internal sealed class RelationNameConverter : JsonConverter<RelationName>
+	{
+		private readonly IElasticsearchClientSettings _settings;
+
+		public RelationNameConverter(IElasticsearchClientSettings settings) => _settings = settings;
+
+		public override RelationName? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => throw new NotImplementedException();
+		public override void Write(Utf8JsonWriter writer, RelationName value, JsonSerializerOptions options)
+		{
+			if (value is null)
+			{
+				writer.WriteNullValue();
+				return;
+			}
+
+			var relationName = _settings.Inferrer.RelationName(value);
+			writer.WriteStringValue(relationName);
+		}
+	}
+
 	internal class FieldResolver
 	{
 		protected readonly ConcurrentDictionary<Field, string> Fields = new ConcurrentDictionary<Field, string>();

src/Elastic.Clients.Elasticsearch/Common/Infer/JoinFieldRouting/Routing.cs

@@ -95,7 +95,7 @@ public static implicit operator Routing(string[] routing) =>
 		/// <summary> Use the inferred routing from <paramref name="document" /> </summary>
 		public static Routing From<T>(T document) where T : class => new(document);
 
-		private string GetString(IElasticsearchClientSettings settings)
+		internal string GetString(IElasticsearchClientSettings settings)
 		{
 			string value = null;
 			if (DocumentGetter != null)
@@ -193,6 +193,13 @@ public override void Write(Utf8JsonWriter writer, Routing value, JsonSerializerO
 			if (value.Document is not null)
 			{
 				var documentId = _settings.Inferrer.Routing(value.Document.GetType(), value.Document);
+
+				if (documentId is null)
+				{
+					writer.WriteNullValue();
+					return;
+				}
+
 				writer.WriteStringValue(documentId);
 			}
 			else if (value.DocumentGetter is not null)

src/Elastic.Clients.Elasticsearch/Serialization/DefaultHighLevelSerializer.cs

@@ -49,6 +49,7 @@ public DefaultHighLevelSerializer(IElasticsearchClientSettings settings)
 					new IdConverter(settings),
 					new FieldConverter(settings),
 					new SortCollectionConverter(settings),
+					new RelationNameConverter(settings),
 					//new FieldNameQueryConverterFactory(settings),
 					new CustomJsonWriterConverterFactory(settings),
 					new RoutingConverter(settings),

src/Elastic.Clients.Elasticsearch/Types/Bulk/BulkOperationDescriptorBase.cs

@@ -74,8 +74,15 @@ protected override void Serialize(Utf8JsonWriter writer, JsonSerializerOptions o
 
 			if (RoutingValue is not null)
 			{
-				writer.WritePropertyName("routing");
-				JsonSerializer.Serialize(writer, RoutingValue, options);
+				// TODO - This flow is a bit inefficient and annoying just to get "clean" JSON
+
+				var value = RoutingValue.GetString(settings);
+
+				if (!string.IsNullOrEmpty(value))
+				{
+					writer.WritePropertyName("routing");
+					JsonSerializer.Serialize(writer, value, options);
+				}
 			}
 
 			if (_version.HasValue)

tests/Tests.Core/Extensions/SerializationTesterAssertionExtensions.cs

@@ -3,6 +3,8 @@
 //// See the LICENSE file in the project root for more information
 
 using System;
+using System.Collections.Generic;
+using System.Text.Json;
 using Tests.Core.Serialization;
 
 namespace Tests.Core.Extensions
@@ -49,5 +51,22 @@ public static void AssertSerialize<T>(this SerializationTester tester, T @object
 			var roundTripResult = tester.Serializes(@object, expectedJson, preserveNullInExpected);
 			roundTripResult.ShouldBeValid(message);
 		}
+
+		public static void AssertSerializeNdjson<T>(this SerializationTester tester, T @object, IReadOnlyList<object> expectedJson, string message = null,
+			bool preserveNullInExpected = false
+		)
+		{
+			var roundTripResult = tester.SerializesNdJson(@object, expectedJson, preserveNullInExpected);
+			roundTripResult.ShouldBeValid(message);
+		}
+
+		public static void AssertSerialize<T>(this SerializationTester tester, string actualJsonString, object expectedJson, string message = null,
+			bool preserveNullInExpected = false
+		)
+		{
+			var roundTripResult = tester.Serializes(actualJsonString, expectedJson, preserveNullInExpected);
+
+			roundTripResult.ShouldBeValid(message);
+		}
 	}
 }

tests/Tests.Core/Serialization/ExpectJsonTestBase.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Collections.Generic;
 using Elastic.Clients.Elasticsearch;
 using Tests.Core.Extensions;
 using Xunit;
@@ -13,10 +14,20 @@ public abstract class ExpectJsonTestBase
 		protected ExpectJsonTestBase(IElasticClient client) => Tester = new SerializationTester(client);
 
 		protected abstract object ExpectJson { get; }
+		protected abstract IReadOnlyList<object> ExpectNdjson { get; }
+
 		protected virtual bool IncludeNullInExpected => true;
 		protected virtual bool SupportsDeserialization => true; //TODO Validate all overrides for false whether they truly do not support deserialization
 		protected SerializationTester Tester { get; }
 
+		protected void SerializesNdjson<T>(T @object)
+		{
+			if (@object is null || ExpectNdjson is null)
+				return;
+
+			Tester.AssertSerializeNdjson(@object, ExpectNdjson, preserveNullInExpected: IncludeNullInExpected);
+		}
+
 		protected void RoundTripsOrSerializes<T>(T @object)
 		{
 			if (@object is null || ExpectJson is null)

tests/Tests.Core/Serialization/SerializationTester.cs

@@ -132,6 +132,39 @@ public SerializationResult Serializes<T>(T @object, object expectedJson, bool pr
 			return result;
 		}
 
+		public SerializationResult SerializesNdJson<T>(T @object, IReadOnlyList<object> expectedJson, bool preserveNullInExpected = false)
+		{
+			var stream = new MemoryStream();
+
+			Serializer.Serialize(@object, stream);
+
+			stream.Position = 0;
+			var reader = new StreamReader(stream);
+
+			var counter = 0;
+			string line;
+
+			var finalResult = new RoundTripResult<T> { Success = true };
+
+			while ((line = reader.ReadLine()) is not null)
+			{
+				var result = new RoundTripResult<T> { Success = false };
+				result.Serialized = line;
+
+				var expected = expectedJson[counter];
+				var expectedJsonToken = ExpectedJsonToJsonDocument(expected, preserveNullInExpected);
+
+				if (!TokenMatches(expectedJsonToken, result))
+				{
+					finalResult.Success = false;
+				}
+
+				counter++;
+			}
+
+			return finalResult;
+		}
+
 		public DeserializationResult<T> Deserializes<T>(object expectedJson, bool preserveNullInExpected = false)
 		{
 			var expectedJsonString = ExpectedJsonString(expectedJson, preserveNullInExpected);
@@ -187,6 +220,8 @@ private bool SerializesAndMatches<T>(T @object, JsonDocument expectedJsonDocumen
 			//	: TokenMatches(expectedJsonDocument, result);
 		}
 
+		
+
 		private string SerializeUsingClientDefault<T>(T @object) =>
 			@object switch
 			{

tests/Tests.Domain/Project.cs

@@ -9,6 +9,8 @@
 using Tests.Domain.Helpers;
 using System.Linq;
 using Elastic.Clients.Elasticsearch;
+using Elastic.Transport;
+using Tests.Domain.Extensions;
 
 namespace Tests.Domain;
 
@@ -27,7 +29,7 @@ public class Project
 		? InstanceAnonymousSourceSerializer
 		: InstanceAnonymousDefault;
 
-	//public JoinField Join => JoinField.Root<Project>();
+	public JoinField Join => JoinField.Root<Project>();
 
 	public Labels Labels { get; set; }
 	public DateTime LastActivity { get; set; }
@@ -126,7 +128,7 @@ public class Project
 	{
 		name = First.Name,
 		type = TypeName,
-		//join = Instance.Join.ToAnonymousObject(),
+		join = Instance.Join.ToAnonymousObject(),
 		state = "BellyUp",
 		visibility = "Public",
 		startedOn = "2015-01-01T00:00:00",
@@ -141,7 +143,7 @@ public class Project
 	{
 		name = First.Name,
 		type = TypeName,
-		//join = Instance.Join.ToAnonymousObject(),
+		join = Instance.Join.ToAnonymousObject(),
 		state = "BellyUp",
 		visibility = "Public",
 		startedOn = "2015-01-01T00:00:00",
@@ -156,3 +158,15 @@ public class Project
 
 	// @formatter:on — enable formatter after this line
 }
+
+public static class AnonymizerExtensions
+{
+	private static readonly Inferrer Infer = new Inferrer(new ElasticsearchClientSettings(new InMemoryConnection()).ApplyDomainSettings());
+
+	public static object ToAnonymousObject(this JoinField field) =>
+		field.Match<object>(p => Infer.RelationName(p.Name), c => new
+		{
+			parent = Infer.Id(c.ParentId),
+			name = Infer.RelationName(c.Name)
+		});
+}