Backport 8.18 Register TypeInfoProvider explicitly

examples/playground/Program.cs

@@ -1,5 +1,6 @@
 using Elastic.Channels;
 using Elastic.CommonSchema;
+using Elastic.CommonSchema.Serialization;
 using Elastic.Elasticsearch.Ephemeral;
 using Elastic.Ingest.Elasticsearch;
 using Elastic.Ingest.Elasticsearch.CommonSchema;
@@ -116,7 +117,8 @@ EcsDataStreamChannel<EcsDocument> SetupElasticsearchChannel()
 	var c = new EcsDataStreamChannel<EcsDocument>(
 		new DataStreamChannelOptions<EcsDocument>(new DistributedTransport(transportConfiguration))
 		{
-			BufferOptions = bufferOptions
+			BufferOptions = bufferOptions,
+			SerializerContext = EcsJsonContext.Default
 		});
 
 	return c;

src/Elastic.CommonSchema.BenchmarkDotNetExporter/ElasticsearchBenchmarkExporter.cs

@@ -13,6 +13,7 @@
 using BenchmarkDotNet.Reports;
 using Elastic.Channels;
 using Elastic.CommonSchema.BenchmarkDotNetExporter.Domain;
+using Elastic.CommonSchema.Serialization;
 using Elastic.Ingest.Elasticsearch.CommonSchema;
 using Elastic.Ingest.Elasticsearch.DataStreams;
 using Elastic.Transport;
@@ -69,6 +70,7 @@ public override void ExportToLog(Summary summary, ILogger logger)
 					OutboundBufferMaxSize = benchmarksCount,
 					OutboundBufferMaxLifetime = TimeSpan.FromSeconds(5)
 				},
+				SerializerContext = EcsJsonContext.Default,
 				ExportExceptionCallback = e => observedException ??= e,
 				ExportResponseCallback = (response, _) =>
 				{

src/Elastic.CommonSchema.NLog/EcsLayout.cs

@@ -16,20 +16,11 @@
 
 namespace Elastic.CommonSchema.NLog
 {
-	internal class NlogEcsDocumentCreationOptions : IEcsDocumentCreationOptions
-	{
-		public static NlogEcsDocumentCreationOptions Default { get; } = new();
-		public bool IncludeHost { get; set; } = true;
-		public bool IncludeProcess { get; set; } = false;
-		public bool IncludeUser { get; set; } = false;
-		public bool IncludeActivityData { get; set; } = false;
-	}
-
 	/// <summary> An NLOG layout implementation that renders logs as ECS json</summary>
 	[Layout(Name)]
 	[ThreadSafe]
 	[ThreadAgnostic]
-	public class EcsLayout : Layout
+	public partial class EcsLayout : Layout
 	{
 		/// <summary> An NLOG layout implementation that renders logs as ECS json</summary>
 		public const string Name = nameof(EcsLayout);
@@ -575,7 +566,7 @@ private Process GetProcess(LogEventInfo logEventInfo)
 			var processExecutable = ProcessExecutable?.Render(logEventInfo);
 			var processThreadId = ProcessThreadId?.Render(logEventInfo);
 			var processThreadName = ProcessThreadName?.Render(logEventInfo);
-			
+
 			var previousProcess = _previousProcess;
 			if (string.IsNullOrEmpty(processThreadId) && string.IsNullOrEmpty(processThreadName))
 			{

src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace Elastic.CommonSchema.NLog;
+
+/// <summary>
+///
+/// </summary>
+[JsonSerializable(typeof(EcsLayout.NLogEcsDocument))]
+[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
+public partial class NLogEcsJsonContext : JsonSerializerContext { }

src/Elastic.CommonSchema.NLog/NlogEcsDocumentCreationOptions.cs

@@ -0,0 +1,10 @@
+namespace Elastic.CommonSchema.NLog;
+
+internal class NlogEcsDocumentCreationOptions : IEcsDocumentCreationOptions
+{
+	public static NlogEcsDocumentCreationOptions Default { get; } = new();
+	public bool IncludeHost { get; set; } = true;
+	public bool IncludeProcess { get; set; } = false;
+	public bool IncludeUser { get; set; } = false;
+	public bool IncludeActivityData { get; set; } = false;
+}

src/Elastic.CommonSchema/EcsDocument.Serialization.cs

@@ -63,7 +63,7 @@ protected internal virtual void WriteAdditionalProperties(Action<string, object>
 		public static EcsDocument? Deserialize(Stream stream) => EcsSerializerFactory<EcsDocument>.Deserialize(stream);
 
 		// ReSharper disable once UnusedMember.Global
-		/// <summary> Deserialize asynchronously a stream to <see cref="EcsDocument"/> </summary>
+		/// <summary> Asynchronously deserialize a stream to <see cref="EcsDocument"/> </summary>
 		public static ValueTask<EcsDocument?> DeserializeAsync(Stream stream, CancellationToken ctx = default) =>
 			EcsSerializerFactory<EcsDocument>.DeserializeAsync(stream, ctx);
 

src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs

@@ -6,14 +6,17 @@
 /*
 IMPORTANT NOTE
 ==============
-This file has been generated. 
+This file has been generated.
 If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks!
 */
 
 using System.Text.Json.Serialization;
 
 namespace Elastic.CommonSchema.Serialization;
 
+/// An implementation of <see cref="JsonSerializerContext"/> that could be used to be combined with user data
+[JsonSerializable(typeof(EcsDocument))]
+[JsonSerializable(typeof(Labels))]
 [JsonSerializable(typeof(Agent))]
 [JsonSerializable(typeof(As))]
 [JsonSerializable(typeof(Client))]
@@ -63,7 +66,7 @@ namespace Elastic.CommonSchema.Serialization;
 [JsonSerializable(typeof(Vlan))]
 [JsonSerializable(typeof(Vulnerability))]
 [JsonSerializable(typeof(X509))]
-[JsonSerializable(typeof(LogEntityJsonConverter.LogOriginInvalid))]
-[JsonSerializable(typeof(LogEntityJsonConverter.LogFileOriginInvalid))]
+[JsonSerializable(typeof(ParserIntermediary.LogOriginInvalid))]
+[JsonSerializable(typeof(ParserIntermediary.LogFileOriginInvalid))]
 [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
-internal partial class EcsJsonContext : JsonSerializerContext { }
+public partial class EcsJsonContext : JsonSerializerContext { }

src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs

@@ -6,6 +6,7 @@
 using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.Json.Serialization.Metadata;
 
 namespace Elastic.CommonSchema.Serialization
 {
@@ -19,6 +20,7 @@ public static class EcsJsonConfiguration
 			NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals,
 			Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
 			PropertyNamingPolicy = new SnakeCaseJsonNamingPolicy(),
+			TypeInfoResolver = JsonTypeInfoResolver.Combine(new DefaultJsonTypeInfoResolver(), EcsJsonContext.Default),
 			Converters =
 			{
 				new EcsDocumentJsonConverterFactory(),

src/Elastic.CommonSchema/Serialization/PropertiesReaderJsonConverterBase.cs

@@ -57,29 +57,41 @@ internal partial class EcsEntityJsonConverter
 	private partial bool ReadProperty(ref Utf8JsonReader reader, string propertyName, Ecs ecsEvent, JsonSerializerOptions options) => false;
 }
 
-internal partial class LogEntityJsonConverter
+
+/// intermediary parser objects, not meant for direct consumption
+public class ParserIntermediary
 {
-	internal class LogFileOriginInvalid
+	/// intermediary parser for log data in a different format
+	public class LogFileOriginInvalid
 	{
+		/// <inheritdoc cref="LogFieldSet.OriginFileName"/>
 		[JsonPropertyName("name"), DataMember(Name = "name")]
 		public string? Name { get; set; }
 
+		/// <inheritdoc cref="LogFieldSet.OriginFileLine"/>
 		[JsonPropertyName("line"), DataMember(Name = "line")]
 		public int? Line { get; set; }
 	}
-	internal class LogOriginInvalid
+
+	/// intermediary parser for log data in a different format
+	public class LogOriginInvalid
 	{
+		/// <inheritdoc cref="LogFieldSet.OriginFunction"/>
 		[JsonPropertyName("function"), DataMember(Name = "function")]
 		public string? Function { get; set; }
 
+		/// <inheritdoc cref="LogFieldSet.FilePath"/>
 		[JsonPropertyName("file"), DataMember(Name = "file")]
 		public LogFileOriginInvalid? File { get; set; }
 	}
+}
 
+internal partial class LogEntityJsonConverter
+{
 	private partial bool ReadProperty(ref Utf8JsonReader reader, string propertyName, Log ecsEvent, JsonSerializerOptions options) =>
 		propertyName switch
 		{
-			"origin" => ReadProp<LogOriginInvalid>(ref reader, "origin", ecsEvent, (b, v) =>
+			"origin" => ReadProp<ParserIntermediary.LogOriginInvalid>(ref reader, "origin", ecsEvent, (b, v) =>
 			{
 				if (v == null) return;
 				b.OriginFunction = v.Function;

src/Elastic.Extensions.Logging.Common/LogEvent.cs

@@ -81,4 +81,5 @@ protected override void WriteAdditionalProperties(Action<string, object> write)
 			if (Scopes != null) write(ScopesPropertyName, Scopes);
 		}
 	}
+
 }

src/Elastic.Extensions.Logging.Common/LogEventJsonContext.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace Elastic.Extensions.Logging.Common;
+
+/// <summary>
+///
+/// </summary>
+[JsonSerializable(typeof(LogEvent))]
+[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
+public partial class LogEventJsonContext : JsonSerializerContext { }

src/Elastic.Extensions.Logging/ElasticsearchLoggerOptionsSetup.cs

@@ -6,13 +6,7 @@
 using Microsoft.Extensions.Logging.Configuration;
 using Microsoft.Extensions.Options;
 
-namespace Elastic.Extensions.Logging
-{
-	internal class ElasticsearchLoggerOptionsSetup : ConfigureFromConfigurationOptions<ElasticsearchLoggerOptions>
-	{
-		public ElasticsearchLoggerOptionsSetup(
-			ILoggerProviderConfiguration<ElasticsearchLoggerProvider> providerConfiguration
-		)
-			: base(providerConfiguration.Configuration) { }
-	}
-}
+namespace Elastic.Extensions.Logging;
+
+internal class ElasticsearchLoggerOptionsSetup(ILoggerProviderConfiguration<ElasticsearchLoggerProvider> providerConfiguration)
+	: ConfigureFromConfigurationOptions<ElasticsearchLoggerOptions>(providerConfiguration.Configuration);

src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs

@@ -6,10 +6,13 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json.Serialization.Metadata;
 using System.Threading;
 using System.Threading.Tasks;
 using Elastic.Channels;
 using Elastic.Channels.Diagnostics;
+using Elastic.CommonSchema.Serialization;
+using Elastic.Extensions.Logging.Common;
 using Elastic.Extensions.Logging.Options;
 using Elastic.Ingest.Elasticsearch;
 using Elastic.Ingest.Elasticsearch.CommonSchema;
@@ -189,6 +192,7 @@ private IBufferedChannel<LogEvent> CreatIngestChannel(ElasticsearchLoggerOptions
 					IndexFormat = loggerOptions.Index.Format,
 					IndexOffset = loggerOptions.Index.IndexOffset,
 					TimestampLookup = l => l.Timestamp,
+					SerializerContexts = [LogEventJsonContext.Default, EcsJsonContext.Default]
 				};
 				SetupChannelOptions(_channelConfigurations, indexChannelOptions);
 				return new EcsIndexChannel<LogEvent>(indexChannelOptions);
@@ -200,7 +204,8 @@ private IBufferedChannel<LogEvent> CreatIngestChannel(ElasticsearchLoggerOptions
 				var indexChannelOptions = new DataStreamChannelOptions<LogEvent>(transport)
 				{
 					DataStream = new DataStreamName(dataStreamNameOptions.Type, dataStreamNameOptions.DataSet, dataStreamNameOptions.Namespace),
-					EventWriter = LogEventWriterInstance
+					EventWriter = LogEventWriterInstance,
+					SerializerContexts = [LogEventJsonContext.Default, EcsJsonContext.Default]
 				};
 
 				SetupChannelOptions(_channelConfigurations, indexChannelOptions);

src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj

@@ -10,7 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Elastic.Ingest.Elasticsearch" Version="0.12.1" />
+    <PackageReference Include="Elastic.Ingest.Elasticsearch" Version="0.12.3" />
   </ItemGroup>
 
 </Project>

src/Elastic.NLog.Targets/ElasticsearchTarget.cs

@@ -1,10 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.Json.Serialization.Metadata;
 using Elastic.Channels;
 using Elastic.Channels.Buffers;
 using Elastic.Channels.Diagnostics;
 using Elastic.CommonSchema.NLog;
+using Elastic.CommonSchema.Serialization;
 using Elastic.Ingest.Elasticsearch;
 using Elastic.Ingest.Elasticsearch.CommonSchema;
 using Elastic.Ingest.Elasticsearch.DataStreams;
@@ -223,13 +225,9 @@ protected override void InitializeTarget()
 
 			var transport = new DistributedTransport<ITransportConfiguration>(config);
 			if (!string.IsNullOrEmpty(indexFormat))
-			{
 				_channel = CreateIndexChannel(transport, indexFormat, indexOffset, IndexOperation);
-			}
 			else
-			{
 				_channel = CreateDataStreamChannel(transport);
-			}
 		}
 
 		private void SetupChannelOptions(ElasticsearchChannelOptionsBase<NLogEcsDocument> channelOptions)
@@ -255,7 +253,8 @@ private EcsDataStreamChannel<NLogEcsDocument> CreateDataStreamChannel(Distribute
 			var dataStreamNamespace = DataStreamNamespace?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty;
 			var channelOptions = new DataStreamChannelOptions<NLogEcsDocument>(transport)
 			{
-				DataStream = new DataStreamName(dataStreamType, dataStreamSet, dataStreamNamespace)
+				DataStream = new DataStreamName(dataStreamType, dataStreamSet, dataStreamNamespace),
+				SerializerContexts = [EcsJsonContext.Default, Elastic.CommonSchema.NLog.NLogEcsJsonContext.Default]
 			};
 			SetupChannelOptions(channelOptions);
 			var channel = new EcsDataStreamChannel<NLogEcsDocument>(channelOptions, new[] { new InternalLoggerCallbackListener<NLogEcsDocument>() });
@@ -270,13 +269,11 @@ private EcsIndexChannel<NLogEcsDocument> CreateIndexChannel(DistributedTransport
 				IndexFormat = indexFormat,
 				IndexOffset = indexOffset,
 				TimestampLookup = l => l.Timestamp,
-				OperationMode = indexOperation
+				OperationMode = indexOperation,
+				SerializerContexts = [EcsJsonContext.Default, Elastic.CommonSchema.NLog.NLogEcsJsonContext.Default]
 			};
 
-			if (_hasIndexEventId)
-			{
-				indexChannelOptions.BulkOperationIdLookup = (logEvent) => (logEvent.Event?.Id)!;
-			}
+			if (_hasIndexEventId) indexChannelOptions.BulkOperationIdLookup = (logEvent) => (logEvent.Event?.Id)!;
 
 			SetupChannelOptions(indexChannelOptions);
 			return new EcsIndexChannel<NLogEcsDocument>(indexChannelOptions);
@@ -290,10 +287,24 @@ protected override void CloseTarget()
 		}
 
 		/// <inheritdoc />
-		protected override void Write(LogEventInfo logEvent)
+		protected override void Write(NLog.Common.AsyncLogEventInfo logEvent)
 		{
-			var ecsDoc = _layout.RenderEcsDocument(logEvent);
-			_channel?.TryWrite(ecsDoc);
+			try
+			{
+				var ecsDoc = _layout.RenderEcsDocument(logEvent.LogEvent);
+				if (_channel?.TryWrite(ecsDoc) == true)
+					logEvent.Continuation(null);
+				else
+				{
+					NLog.Common.InternalLogger.Error("ElasticSearch - Failed writing to Elastic channel (Buffer full)");
+					logEvent.Continuation(new System.Threading.Tasks.TaskCanceledException("Failed writing to Elastic channel (Buffer full)"));
+				}
+			}
+			catch (Exception ex)
+			{
+				logEvent.Continuation(ex);
+				throw;
+			}
 		}
 
 		private NodePool CreateNodePool()
@@ -358,18 +369,18 @@ private TransportConfigurationDescriptor SetAuthenticationOnTransport(TransportC
 	{
 		public Action<Exception>? ExportExceptionCallback { get; }
 		public Action<BulkResponse, IWriteTrackingBuffer>? ExportResponseCallback { get; }
+		public Action? PublishToInboundChannelFailureCallback { get; }
+		public Action? PublishToOutboundChannelFailureCallback { get; }
 
 		// ReSharper disable UnassignedGetOnlyAutoProperty
 		public Action<int, int>? ExportItemsAttemptCallback { get; }
 		public Action<IReadOnlyCollection<TNLogEcsDocument>>? ExportMaxRetriesCallback { get; }
 		public Action<IReadOnlyCollection<TNLogEcsDocument>>? ExportRetryCallback { get; }
 		public Action? PublishToInboundChannelCallback { get; }
-		public Action? PublishToInboundChannelFailureCallback { get; }
 		public Action? PublishToOutboundChannelCallback { get; }
 		public Action? OutboundChannelStartedCallback { get; }
 		public Action? OutboundChannelExitedCallback { get; }
 		public Action? InboundChannelStartedCallback { get; }
-		public Action? PublishToOutboundChannelFailureCallback { get; }
 		public Action? ExportBufferCallback { get; }
 		public Action<int>? ExportRetryableCountCallback { get; }
 		// ReSharper enable UnassignedGetOnlyAutoProperty
@@ -380,6 +391,14 @@ public InternalLoggerCallbackListener()
 			{
 				NLog.Common.InternalLogger.Error(ex, "ElasticSearch - Export Exception");
 			};
+			PublishToInboundChannelFailureCallback = () =>
+			{
+				NLog.Common.InternalLogger.Error("ElasticSearch - Inbound Channel Failure (Buffer full)");
+			};
+			PublishToOutboundChannelFailureCallback = () =>
+			{
+				NLog.Common.InternalLogger.Error("ElasticSearch - Outbound Channel Failure (Flush failed)");
+			};
 			ExportResponseCallback = (response, _) =>
 			{
 				if (response is null)

src/Elastic.Serilog.Sinks/ElasticsearchSink.cs

@@ -5,6 +5,7 @@
 using Elastic.Channels.Buffers;
 using Elastic.Channels.Diagnostics;
 using Elastic.CommonSchema;
+using Elastic.CommonSchema.Serialization;
 using Elastic.CommonSchema.Serilog;
 using Elastic.Ingest.Elasticsearch;
 using Elastic.Ingest.Elasticsearch.CommonSchema;
@@ -136,7 +137,8 @@ public ElasticsearchSink(ElasticsearchSinkOptions<TEcsDocument> options)
 			var channelOptions = new DataStreamChannelOptions<TEcsDocument>(options.Transport)
 			{
 				DataStream = options.DataStream,
-				ExportMaxRetriesCallback = EmitExportFailures
+				ExportMaxRetriesCallback = EmitExportFailures,
+				SerializerContext = EcsJsonContext.Default,
 			};
 			options.ConfigureChannel?.Invoke(channelOptions);
 			_channel = new EcsDataStreamChannel<TEcsDocument>(channelOptions, new [] { new SelfLogCallbackListener<TEcsDocument>(options)});