From 512a7074118e94e70c4224bcd8cc48c204089648 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 4 Aug 2025 18:49:43 +0200 Subject: [PATCH 1/8] Add EcsJsonContext.Default to JsonSerializerOptions --- src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs | 1 + .../Views/EcsJsonContext.Generated.cshtml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs b/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs index 7facf016..311193f4 100644 --- a/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs +++ b/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs @@ -19,6 +19,7 @@ public static class EcsJsonConfiguration NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, PropertyNamingPolicy = new SnakeCaseJsonNamingPolicy(), + TypeInfoResolver = EcsJsonContext.Default, Converters = { new EcsDocumentJsonConverterFactory(), diff --git a/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml index 88363fad..195a5c50 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml @@ -17,6 +17,8 @@ using System.Text.Json.Serialization; namespace Elastic.CommonSchema.Serialization; +[JsonSerializable(typeof(EcsDocument))] +[JsonSerializable(typeof(Labels))] @foreach (var property in Model.Base.EntityProperties) { [JsonSerializable(typeof(@property.Entity.Name))] @@ -30,4 +32,4 @@ namespace Elastic.CommonSchema.Serialization; [JsonSerializable(typeof(LogEntityJsonConverter.LogOriginInvalid))] [JsonSerializable(typeof(LogEntityJsonConverter.LogFileOriginInvalid))] [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] -internal partial class EcsJsonContext : JsonSerializerContext { } \ No newline at end of file +internal partial class EcsJsonContext : JsonSerializerContext { } From 271652ae3e0109eb6f2d4a09d03cf5b0e13058b6 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 4 Aug 2025 18:56:18 +0200 Subject: [PATCH 2/8] Update JsonSerializerOptions to combine DefaultJsonTypeInfoResolver with EcsJsonContext --- src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs b/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs index 311193f4..549758aa 100644 --- a/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs +++ b/src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs @@ -6,6 +6,7 @@ using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; namespace Elastic.CommonSchema.Serialization { @@ -19,7 +20,7 @@ public static class EcsJsonConfiguration NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, PropertyNamingPolicy = new SnakeCaseJsonNamingPolicy(), - TypeInfoResolver = EcsJsonContext.Default, + TypeInfoResolver = JsonTypeInfoResolver.Combine(new DefaultJsonTypeInfoResolver(), EcsJsonContext.Default), Converters = { new EcsDocumentJsonConverterFactory(), From 5350e4ace11d14071c1d2a299b5555885fecdda9 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 4 Aug 2025 18:56:29 +0200 Subject: [PATCH 3/8] Add JsonSerializable attributes for EcsDocument and Labels in EcsJsonContext --- .../Serialization/EcsJsonContext.Generated.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs b/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs index 60458686..92bf6dea 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs @@ -14,6 +14,8 @@ If you wish to submit a PR please modify the original csharp file and submit the namespace Elastic.CommonSchema.Serialization; +[JsonSerializable(typeof(EcsDocument))] +[JsonSerializable(typeof(Labels))] [JsonSerializable(typeof(Agent))] [JsonSerializable(typeof(As))] [JsonSerializable(typeof(Client))] @@ -67,4 +69,4 @@ namespace Elastic.CommonSchema.Serialization; [JsonSerializable(typeof(LogEntityJsonConverter.LogOriginInvalid))] [JsonSerializable(typeof(LogEntityJsonConverter.LogFileOriginInvalid))] [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] -internal partial class EcsJsonContext : JsonSerializerContext { } \ No newline at end of file +internal partial class EcsJsonContext : JsonSerializerContext { } From c50d1167746c8366343f0b6f351cab5c5caa8473 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 4 Aug 2025 19:22:07 +0200 Subject: [PATCH 4/8] Add `EcsJsonContext.Default` usage for serialization across channels and update parser intermediary types --- examples/playground/Program.cs | 4 +++- .../ElasticsearchBenchmarkExporter.cs | 2 ++ .../PropertiesReaderJsonConverterBase.cs | 20 +++++++++++++++---- .../ElasticsearchLoggerProvider.cs | 5 ++++- .../ElasticsearchTarget.cs | 7 +++++-- .../ElasticsearchSink.cs | 6 ++---- .../DataStreamIngestionTests.cs | 5 ++++- .../IndexIngestionTests.cs | 3 +++ .../Views/EcsJsonContext.Generated.cshtml | 7 ++++--- 9 files changed, 43 insertions(+), 16 deletions(-) diff --git a/examples/playground/Program.cs b/examples/playground/Program.cs index 49c62596..83af6956 100644 --- a/examples/playground/Program.cs +++ b/examples/playground/Program.cs @@ -1,5 +1,6 @@ using Elastic.Channels; using Elastic.CommonSchema; +using Elastic.CommonSchema.Serialization; using Elastic.Elasticsearch.Ephemeral; using Elastic.Ingest.Elasticsearch; using Elastic.Ingest.Elasticsearch.CommonSchema; @@ -116,7 +117,8 @@ EcsDataStreamChannel SetupElasticsearchChannel() var c = new EcsDataStreamChannel( new DataStreamChannelOptions(new DistributedTransport(transportConfiguration)) { - BufferOptions = bufferOptions + BufferOptions = bufferOptions, + SerializerContext = EcsJsonContext.Default }); return c; diff --git a/src/Elastic.CommonSchema.BenchmarkDotNetExporter/ElasticsearchBenchmarkExporter.cs b/src/Elastic.CommonSchema.BenchmarkDotNetExporter/ElasticsearchBenchmarkExporter.cs index 923a289d..df360a8b 100644 --- a/src/Elastic.CommonSchema.BenchmarkDotNetExporter/ElasticsearchBenchmarkExporter.cs +++ b/src/Elastic.CommonSchema.BenchmarkDotNetExporter/ElasticsearchBenchmarkExporter.cs @@ -13,6 +13,7 @@ using BenchmarkDotNet.Reports; using Elastic.Channels; using Elastic.CommonSchema.BenchmarkDotNetExporter.Domain; +using Elastic.CommonSchema.Serialization; using Elastic.Ingest.Elasticsearch.CommonSchema; using Elastic.Ingest.Elasticsearch.DataStreams; using Elastic.Transport; @@ -69,6 +70,7 @@ public override void ExportToLog(Summary summary, ILogger logger) OutboundBufferMaxSize = benchmarksCount, OutboundBufferMaxLifetime = TimeSpan.FromSeconds(5) }, + SerializerContext = EcsJsonContext.Default, ExportExceptionCallback = e => observedException ??= e, ExportResponseCallback = (response, _) => { diff --git a/src/Elastic.CommonSchema/Serialization/PropertiesReaderJsonConverterBase.cs b/src/Elastic.CommonSchema/Serialization/PropertiesReaderJsonConverterBase.cs index 5a24fd74..5664a863 100644 --- a/src/Elastic.CommonSchema/Serialization/PropertiesReaderJsonConverterBase.cs +++ b/src/Elastic.CommonSchema/Serialization/PropertiesReaderJsonConverterBase.cs @@ -57,29 +57,41 @@ internal partial class EcsEntityJsonConverter private partial bool ReadProperty(ref Utf8JsonReader reader, string propertyName, Ecs ecsEvent, JsonSerializerOptions options) => false; } -internal partial class LogEntityJsonConverter + +/// intermediary parser objects, not meant for direct consumption +public class ParserIntermediary { - internal class LogFileOriginInvalid + /// intermediary parser for log data in a different format + public class LogFileOriginInvalid { + /// [JsonPropertyName("name"), DataMember(Name = "name")] public string? Name { get; set; } + /// [JsonPropertyName("line"), DataMember(Name = "line")] public int? Line { get; set; } } - internal class LogOriginInvalid + + /// intermediary parser for log data in a different format + public class LogOriginInvalid { + /// [JsonPropertyName("function"), DataMember(Name = "function")] public string? Function { get; set; } + /// [JsonPropertyName("file"), DataMember(Name = "file")] public LogFileOriginInvalid? File { get; set; } } +} +internal partial class LogEntityJsonConverter +{ private partial bool ReadProperty(ref Utf8JsonReader reader, string propertyName, Log ecsEvent, JsonSerializerOptions options) => propertyName switch { - "origin" => ReadProp(ref reader, "origin", ecsEvent, (b, v) => + "origin" => ReadProp(ref reader, "origin", ecsEvent, (b, v) => { if (v == null) return; b.OriginFunction = v.Function; diff --git a/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs b/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs index 9a73ad48..0511ac75 100644 --- a/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs +++ b/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Elastic.Channels; using Elastic.Channels.Diagnostics; +using Elastic.CommonSchema.Serialization; using Elastic.Extensions.Logging.Options; using Elastic.Ingest.Elasticsearch; using Elastic.Ingest.Elasticsearch.CommonSchema; @@ -189,6 +190,7 @@ private IBufferedChannel CreatIngestChannel(ElasticsearchLoggerOptions IndexFormat = loggerOptions.Index.Format, IndexOffset = loggerOptions.Index.IndexOffset, TimestampLookup = l => l.Timestamp, + SerializerContext = EcsJsonContext.Default, }; SetupChannelOptions(_channelConfigurations, indexChannelOptions); return new EcsIndexChannel(indexChannelOptions); @@ -200,7 +202,8 @@ private IBufferedChannel CreatIngestChannel(ElasticsearchLoggerOptions var indexChannelOptions = new DataStreamChannelOptions(transport) { DataStream = new DataStreamName(dataStreamNameOptions.Type, dataStreamNameOptions.DataSet, dataStreamNameOptions.Namespace), - EventWriter = LogEventWriterInstance + EventWriter = LogEventWriterInstance, + SerializerContext = EcsJsonContext.Default, }; SetupChannelOptions(_channelConfigurations, indexChannelOptions); diff --git a/src/Elastic.NLog.Targets/ElasticsearchTarget.cs b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs index a0120c20..98a31527 100644 --- a/src/Elastic.NLog.Targets/ElasticsearchTarget.cs +++ b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs @@ -5,6 +5,7 @@ using Elastic.Channels.Buffers; using Elastic.Channels.Diagnostics; using Elastic.CommonSchema.NLog; +using Elastic.CommonSchema.Serialization; using Elastic.Ingest.Elasticsearch; using Elastic.Ingest.Elasticsearch.CommonSchema; using Elastic.Ingest.Elasticsearch.DataStreams; @@ -255,7 +256,8 @@ private EcsDataStreamChannel CreateDataStreamChannel(Distribute var dataStreamNamespace = DataStreamNamespace?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; var channelOptions = new DataStreamChannelOptions(transport) { - DataStream = new DataStreamName(dataStreamType, dataStreamSet, dataStreamNamespace) + DataStream = new DataStreamName(dataStreamType, dataStreamSet, dataStreamNamespace), + SerializerContext = EcsJsonContext.Default, }; SetupChannelOptions(channelOptions); var channel = new EcsDataStreamChannel(channelOptions, new[] { new InternalLoggerCallbackListener() }); @@ -270,7 +272,8 @@ private EcsIndexChannel CreateIndexChannel(DistributedTransport IndexFormat = indexFormat, IndexOffset = indexOffset, TimestampLookup = l => l.Timestamp, - OperationMode = indexOperation + OperationMode = indexOperation, + SerializerContext = EcsJsonContext.Default, }; if (_hasIndexEventId) diff --git a/src/Elastic.Serilog.Sinks/ElasticsearchSink.cs b/src/Elastic.Serilog.Sinks/ElasticsearchSink.cs index 4aa0c0e7..ac841dff 100644 --- a/src/Elastic.Serilog.Sinks/ElasticsearchSink.cs +++ b/src/Elastic.Serilog.Sinks/ElasticsearchSink.cs @@ -5,6 +5,7 @@ using Elastic.Channels.Buffers; using Elastic.Channels.Diagnostics; using Elastic.CommonSchema; +using Elastic.CommonSchema.Serialization; using Elastic.CommonSchema.Serilog; using Elastic.Ingest.Elasticsearch; using Elastic.Ingest.Elasticsearch.CommonSchema; @@ -137,10 +138,7 @@ public ElasticsearchSink(ElasticsearchSinkOptions options) { DataStream = options.DataStream, ExportMaxRetriesCallback = EmitExportFailures, - ExportExceptionCallback = _ => - { - - } + SerializerContext = EcsJsonContext.Default, }; options.ConfigureChannel?.Invoke(channelOptions); _channel = new EcsDataStreamChannel(channelOptions, new [] { new SelfLogCallbackListener(options)}); diff --git a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs index 860a354c..377c4f3e 100644 --- a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs +++ b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/DataStreamIngestionTests.cs @@ -6,6 +6,7 @@ using Elastic.Channels; using Elastic.Clients.Elasticsearch.IndexManagement; using Elastic.CommonSchema; +using Elastic.CommonSchema.Serialization; using Elastic.Ingest.Elasticsearch.DataStreams; using Elastic.Transport; using FluentAssertions; @@ -32,6 +33,7 @@ public async Task EnsureDocumentsEndUpInDataStream() { DataStream = targetDataStream, BufferOptions = new BufferOptions { WaitHandle = slim, OutboundBufferMaxSize = 1 }, + SerializerContext = EcsJsonContext.Default, }; var channel = new EcsDataStreamChannel(options); @@ -80,7 +82,8 @@ public async Task UseCustomEventWriter() { DataStream = targetDataStream, BufferOptions = new BufferOptions { WaitHandle = slim, OutboundBufferMaxSize = 1 }, - EventWriter = new CustomEventWriter() + EventWriter = new CustomEventWriter(), + SerializerContext = EcsJsonContext.Default, }; var channel = new EcsDataStreamChannel(options); diff --git a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs index a736a1b8..c4a93a30 100644 --- a/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs +++ b/tests-integration/Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests/IndexIngestionTests.cs @@ -6,6 +6,7 @@ using Elastic.Channels; using Elastic.Clients.Elasticsearch.IndexManagement; using Elastic.CommonSchema; +using Elastic.CommonSchema.Serialization; using Elastic.Ingest.Elasticsearch.Indices; using Elastic.Transport; using FluentAssertions; @@ -35,6 +36,7 @@ public async Task EnsureDocumentsEndUpInIndex() { WaitHandle = slim, ExportMaxConcurrency = 1, }, + SerializerContext = EcsJsonContext.Default, }; var channel = new EcsIndexChannel(options); var bootstrapped = await channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, "7-days-default"); @@ -87,6 +89,7 @@ public async Task UseCustomEventWriter() { WaitHandle = slim, ExportMaxConcurrency = 1, }, + SerializerContext = EcsJsonContext.Default, EventWriter = new CustomEventWriter() }; var channel = new EcsIndexChannel(options); diff --git a/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml b/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml index 195a5c50..484d1b92 100644 --- a/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml +++ b/tools/Elastic.CommonSchema.Generator/Views/EcsJsonContext.Generated.cshtml @@ -17,6 +17,7 @@ using System.Text.Json.Serialization; namespace Elastic.CommonSchema.Serialization; +/// An implementation of that could be used to be combined with user data [JsonSerializable(typeof(EcsDocument))] [JsonSerializable(typeof(Labels))] @foreach (var property in Model.Base.EntityProperties) @@ -29,7 +30,7 @@ namespace Elastic.CommonSchema.Serialization; [JsonSerializable(typeof(@entity.Name))] } -[JsonSerializable(typeof(LogEntityJsonConverter.LogOriginInvalid))] -[JsonSerializable(typeof(LogEntityJsonConverter.LogFileOriginInvalid))] +[JsonSerializable(typeof(ParserIntermediary.LogOriginInvalid))] +[JsonSerializable(typeof(ParserIntermediary.LogFileOriginInvalid))] [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] -internal partial class EcsJsonContext : JsonSerializerContext { } +public partial class EcsJsonContext : JsonSerializerContext { } From 8ac514c9759be08a33593af10763bd8dd48a5341 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 4 Aug 2025 19:22:17 +0200 Subject: [PATCH 5/8] Update EcsJsonContext: modify access modifier, fix intermediary types, add documentation --- .../Serialization/EcsJsonContext.Generated.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs b/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs index 92bf6dea..5be9f4d1 100644 --- a/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs +++ b/src/Elastic.CommonSchema/Serialization/EcsJsonContext.Generated.cs @@ -6,7 +6,7 @@ /* IMPORTANT NOTE ============== -This file has been generated. +This file has been generated. If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks! */ @@ -14,6 +14,7 @@ If you wish to submit a PR please modify the original csharp file and submit the namespace Elastic.CommonSchema.Serialization; +/// An implementation of that could be used to be combined with user data [JsonSerializable(typeof(EcsDocument))] [JsonSerializable(typeof(Labels))] [JsonSerializable(typeof(Agent))] @@ -66,7 +67,7 @@ namespace Elastic.CommonSchema.Serialization; [JsonSerializable(typeof(Volume))] [JsonSerializable(typeof(Vulnerability))] [JsonSerializable(typeof(X509))] -[JsonSerializable(typeof(LogEntityJsonConverter.LogOriginInvalid))] -[JsonSerializable(typeof(LogEntityJsonConverter.LogFileOriginInvalid))] +[JsonSerializable(typeof(ParserIntermediary.LogOriginInvalid))] +[JsonSerializable(typeof(ParserIntermediary.LogFileOriginInvalid))] [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] -internal partial class EcsJsonContext : JsonSerializerContext { } +public partial class EcsJsonContext : JsonSerializerContext { } From ae78cf5c54c21e8c8a72d8ef3677f33fe9e35d48 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 5 Aug 2025 09:21:28 +0200 Subject: [PATCH 6/8] Provide NLogEcsJsonContext.Default --- src/Elastic.CommonSchema.NLog/EcsLayout.cs | 13 ++----------- .../NLogEcsJsonContext.cs | 9 +++++++++ .../NlogEcsDocumentCreationOptions.cs | 10 ++++++++++ ...ic.Ingest.Elasticsearch.CommonSchema.csproj | 2 +- .../ElasticsearchTarget.cs | 18 ++++++------------ .../Elasticsearch.IntegrationDefaults.csproj | 2 +- 6 files changed, 29 insertions(+), 25 deletions(-) create mode 100644 src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs create mode 100644 src/Elastic.CommonSchema.NLog/NlogEcsDocumentCreationOptions.cs diff --git a/src/Elastic.CommonSchema.NLog/EcsLayout.cs b/src/Elastic.CommonSchema.NLog/EcsLayout.cs index d417540a..4713368f 100644 --- a/src/Elastic.CommonSchema.NLog/EcsLayout.cs +++ b/src/Elastic.CommonSchema.NLog/EcsLayout.cs @@ -16,19 +16,10 @@ namespace Elastic.CommonSchema.NLog { - internal class NlogEcsDocumentCreationOptions : IEcsDocumentCreationOptions - { - public static NlogEcsDocumentCreationOptions Default { get; } = new(); - public bool IncludeHost { get; set; } = true; - public bool IncludeProcess { get; set; } = false; - public bool IncludeUser { get; set; } = false; - public bool IncludeActivityData { get; set; } = false; - } - /// An NLOG layout implementation that renders logs as ECS json [Layout(Name)] [ThreadAgnostic] - public class EcsLayout : Layout + public partial class EcsLayout : Layout { /// An NLOG layout implementation that renders logs as ECS json public const string Name = nameof(EcsLayout); @@ -569,7 +560,7 @@ private Process GetProcess(LogEventInfo logEventInfo) var processExecutable = ProcessExecutable?.Render(logEventInfo); var processThreadId = ProcessThreadId?.Render(logEventInfo); var processThreadName = ProcessThreadName?.Render(logEventInfo); - + var previousProcess = _previousProcess; if (string.IsNullOrEmpty(processThreadId) && string.IsNullOrEmpty(processThreadName)) { diff --git a/src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs b/src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs new file mode 100644 index 00000000..231b22d9 --- /dev/null +++ b/src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; +using Elastic.CommonSchema.NLog; + +/// +/// +/// +[JsonSerializable(typeof(EcsLayout.NLogEcsDocument))] +[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +public partial class NLogEcsJsonContext : JsonSerializerContext { } diff --git a/src/Elastic.CommonSchema.NLog/NlogEcsDocumentCreationOptions.cs b/src/Elastic.CommonSchema.NLog/NlogEcsDocumentCreationOptions.cs new file mode 100644 index 00000000..657ec85f --- /dev/null +++ b/src/Elastic.CommonSchema.NLog/NlogEcsDocumentCreationOptions.cs @@ -0,0 +1,10 @@ +namespace Elastic.CommonSchema.NLog; + +internal class NlogEcsDocumentCreationOptions : IEcsDocumentCreationOptions +{ + public static NlogEcsDocumentCreationOptions Default { get; } = new(); + public bool IncludeHost { get; set; } = true; + public bool IncludeProcess { get; set; } = false; + public bool IncludeUser { get; set; } = false; + public bool IncludeActivityData { get; set; } = false; +} diff --git a/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj b/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj index 1787e90a..d50c1c14 100644 --- a/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj +++ b/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/Elastic.NLog.Targets/ElasticsearchTarget.cs b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs index 98a31527..47955547 100644 --- a/src/Elastic.NLog.Targets/ElasticsearchTarget.cs +++ b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Serialization.Metadata; using Elastic.Channels; using Elastic.Channels.Buffers; using Elastic.Channels.Diagnostics; @@ -224,13 +225,9 @@ protected override void InitializeTarget() var transport = new DistributedTransport(config); if (!string.IsNullOrEmpty(indexFormat)) - { _channel = CreateIndexChannel(transport, indexFormat, indexOffset, IndexOperation); - } else - { _channel = CreateDataStreamChannel(transport); - } } private void SetupChannelOptions(ElasticsearchChannelOptionsBase channelOptions) @@ -257,7 +254,7 @@ private EcsDataStreamChannel CreateDataStreamChannel(Distribute var channelOptions = new DataStreamChannelOptions(transport) { DataStream = new DataStreamName(dataStreamType, dataStreamSet, dataStreamNamespace), - SerializerContext = EcsJsonContext.Default, + SerializerContexts = [EcsJsonContext.Default, NLogEcsJsonContext.Default] }; SetupChannelOptions(channelOptions); var channel = new EcsDataStreamChannel(channelOptions, new[] { new InternalLoggerCallbackListener() }); @@ -273,13 +270,10 @@ private EcsIndexChannel CreateIndexChannel(DistributedTransport IndexOffset = indexOffset, TimestampLookup = l => l.Timestamp, OperationMode = indexOperation, - SerializerContext = EcsJsonContext.Default, + SerializerContexts = [EcsJsonContext.Default, NLogEcsJsonContext.Default] }; - if (_hasIndexEventId) - { - indexChannelOptions.BulkOperationIdLookup = (logEvent) => (logEvent.Event?.Id)!; - } + if (_hasIndexEventId) indexChannelOptions.BulkOperationIdLookup = (logEvent) => (logEvent.Event?.Id)!; SetupChannelOptions(indexChannelOptions); return new EcsIndexChannel(indexChannelOptions); @@ -299,9 +293,7 @@ protected override void Write(NLog.Common.AsyncLogEventInfo logEvent) { var ecsDoc = _layout.RenderEcsDocument(logEvent.LogEvent); if (_channel?.TryWrite(ecsDoc) == true) - { logEvent.Continuation(null); - } else { NLog.Common.InternalLogger.Error("ElasticSearch - Failed writing to Elastic channel (Buffer full)"); @@ -418,8 +410,10 @@ public InternalLoggerCallbackListener() if (response.Items?.Count > 0) { foreach (var itemResult in response.Items) + { if (itemResult?.Status >= 300) NLog.Common.InternalLogger.Error("ElasticSearch - Export Item failed to {0} document status {1} - {2}", itemResult.Action, itemResult.Status, itemResult.Error); + } } }; } diff --git a/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj b/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj index a457f5d8..7f393461 100644 --- a/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj +++ b/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj @@ -11,7 +11,7 @@ - + From 5d080569ad7c0b22ec0bc76380d85eeb8baac93d Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 5 Aug 2025 10:08:09 +0200 Subject: [PATCH 7/8] Ensure LogEvent has a serializer context --- .../NLogEcsJsonContext.cs | 3 ++- .../EcsDocument.Serialization.cs | 2 +- src/Elastic.Extensions.Logging.Common/LogEvent.cs | 1 + .../ElasticsearchLoggerOptionsSetup.cs | 14 ++++---------- .../ElasticsearchLoggerProvider.cs | 6 ++++-- ...lastic.Ingest.Elasticsearch.CommonSchema.csproj | 2 +- src/Elastic.NLog.Targets/ElasticsearchTarget.cs | 4 ++-- .../Elasticsearch.IntegrationDefaults.csproj | 2 +- 8 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs b/src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs index 231b22d9..d94783ad 100644 --- a/src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs +++ b/src/Elastic.CommonSchema.NLog/NLogEcsJsonContext.cs @@ -1,5 +1,6 @@ using System.Text.Json.Serialization; -using Elastic.CommonSchema.NLog; + +namespace Elastic.CommonSchema.NLog; /// /// diff --git a/src/Elastic.CommonSchema/EcsDocument.Serialization.cs b/src/Elastic.CommonSchema/EcsDocument.Serialization.cs index d68a6b7a..a61e4683 100644 --- a/src/Elastic.CommonSchema/EcsDocument.Serialization.cs +++ b/src/Elastic.CommonSchema/EcsDocument.Serialization.cs @@ -63,7 +63,7 @@ protected internal virtual void WriteAdditionalProperties(Action public static EcsDocument? Deserialize(Stream stream) => EcsSerializerFactory.Deserialize(stream); // ReSharper disable once UnusedMember.Global - /// Deserialize asynchronously a stream to + /// Asynchronously deserialize a stream to public static ValueTask DeserializeAsync(Stream stream, CancellationToken ctx = default) => EcsSerializerFactory.DeserializeAsync(stream, ctx); diff --git a/src/Elastic.Extensions.Logging.Common/LogEvent.cs b/src/Elastic.Extensions.Logging.Common/LogEvent.cs index 6f8fd21f..beb7811b 100644 --- a/src/Elastic.Extensions.Logging.Common/LogEvent.cs +++ b/src/Elastic.Extensions.Logging.Common/LogEvent.cs @@ -81,4 +81,5 @@ protected override void WriteAdditionalProperties(Action write) if (Scopes != null) write(ScopesPropertyName, Scopes); } } + } diff --git a/src/Elastic.Extensions.Logging/ElasticsearchLoggerOptionsSetup.cs b/src/Elastic.Extensions.Logging/ElasticsearchLoggerOptionsSetup.cs index ad5d0e8c..c018b51b 100644 --- a/src/Elastic.Extensions.Logging/ElasticsearchLoggerOptionsSetup.cs +++ b/src/Elastic.Extensions.Logging/ElasticsearchLoggerOptionsSetup.cs @@ -6,13 +6,7 @@ using Microsoft.Extensions.Logging.Configuration; using Microsoft.Extensions.Options; -namespace Elastic.Extensions.Logging -{ - internal class ElasticsearchLoggerOptionsSetup : ConfigureFromConfigurationOptions - { - public ElasticsearchLoggerOptionsSetup( - ILoggerProviderConfiguration providerConfiguration - ) - : base(providerConfiguration.Configuration) { } - } -} +namespace Elastic.Extensions.Logging; + +internal class ElasticsearchLoggerOptionsSetup(ILoggerProviderConfiguration providerConfiguration) + : ConfigureFromConfigurationOptions(providerConfiguration.Configuration); diff --git a/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs b/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs index 0511ac75..8a91bc96 100644 --- a/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs +++ b/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs @@ -6,11 +6,13 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; using Elastic.Channels; using Elastic.Channels.Diagnostics; using Elastic.CommonSchema.Serialization; +using Elastic.Extensions.Logging.Common; using Elastic.Extensions.Logging.Options; using Elastic.Ingest.Elasticsearch; using Elastic.Ingest.Elasticsearch.CommonSchema; @@ -190,7 +192,7 @@ private IBufferedChannel CreatIngestChannel(ElasticsearchLoggerOptions IndexFormat = loggerOptions.Index.Format, IndexOffset = loggerOptions.Index.IndexOffset, TimestampLookup = l => l.Timestamp, - SerializerContext = EcsJsonContext.Default, + SerializerContexts = [LogEventJsonContext.Default, EcsJsonContext.Default] }; SetupChannelOptions(_channelConfigurations, indexChannelOptions); return new EcsIndexChannel(indexChannelOptions); @@ -203,7 +205,7 @@ private IBufferedChannel CreatIngestChannel(ElasticsearchLoggerOptions { DataStream = new DataStreamName(dataStreamNameOptions.Type, dataStreamNameOptions.DataSet, dataStreamNameOptions.Namespace), EventWriter = LogEventWriterInstance, - SerializerContext = EcsJsonContext.Default, + SerializerContexts = [LogEventJsonContext.Default, EcsJsonContext.Default] }; SetupChannelOptions(_channelConfigurations, indexChannelOptions); diff --git a/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj b/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj index d50c1c14..77714fe7 100644 --- a/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj +++ b/src/Elastic.Ingest.Elasticsearch.CommonSchema/Elastic.Ingest.Elasticsearch.CommonSchema.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/Elastic.NLog.Targets/ElasticsearchTarget.cs b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs index 47955547..5239d04a 100644 --- a/src/Elastic.NLog.Targets/ElasticsearchTarget.cs +++ b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs @@ -254,7 +254,7 @@ private EcsDataStreamChannel CreateDataStreamChannel(Distribute var channelOptions = new DataStreamChannelOptions(transport) { DataStream = new DataStreamName(dataStreamType, dataStreamSet, dataStreamNamespace), - SerializerContexts = [EcsJsonContext.Default, NLogEcsJsonContext.Default] + SerializerContexts = [EcsJsonContext.Default, Elastic.CommonSchema.NLog.NLogEcsJsonContext.Default] }; SetupChannelOptions(channelOptions); var channel = new EcsDataStreamChannel(channelOptions, new[] { new InternalLoggerCallbackListener() }); @@ -270,7 +270,7 @@ private EcsIndexChannel CreateIndexChannel(DistributedTransport IndexOffset = indexOffset, TimestampLookup = l => l.Timestamp, OperationMode = indexOperation, - SerializerContexts = [EcsJsonContext.Default, NLogEcsJsonContext.Default] + SerializerContexts = [EcsJsonContext.Default, Elastic.CommonSchema.NLog.NLogEcsJsonContext.Default] }; if (_hasIndexEventId) indexChannelOptions.BulkOperationIdLookup = (logEvent) => (logEvent.Event?.Id)!; diff --git a/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj b/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj index 7f393461..3b8b052a 100644 --- a/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj +++ b/tests-integration/Elasticsearch.IntegrationDefaults/Elasticsearch.IntegrationDefaults.csproj @@ -11,7 +11,7 @@ - + From 8d596510b4b2e8d74033c955af1b41ec1e0793c9 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 5 Aug 2025 10:14:23 +0200 Subject: [PATCH 8/8] Add missing file --- .../LogEventJsonContext.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/Elastic.Extensions.Logging.Common/LogEventJsonContext.cs diff --git a/src/Elastic.Extensions.Logging.Common/LogEventJsonContext.cs b/src/Elastic.Extensions.Logging.Common/LogEventJsonContext.cs new file mode 100644 index 00000000..639ca79d --- /dev/null +++ b/src/Elastic.Extensions.Logging.Common/LogEventJsonContext.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace Elastic.Extensions.Logging.Common; + +/// +/// +/// +[JsonSerializable(typeof(LogEvent))] +[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +public partial class LogEventJsonContext : JsonSerializerContext { }