Skip to content

Commit 39cf5ee

Browse files
MpdreamzMpdreamz
authored
Set ECS fields from message templates (#229)
This allows most ECS fields to be set directly from message templates e.g Log.Info("{TraceId}", "x"); will override `trace.id` and not assign `metadata.TraceId` to `x`. This only supports ecs fields not part of an array.
1 parent 5d01b87 commit 39cf5ee

33 files changed

+4330
-82
lines changed

‎src/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@
99
</PropertyGroup>
1010

1111
<ItemGroup>
12+
<PackageReference Include="RazorLight" Version="2.3.0" />
1213
<PackageReference Include="Cogito.Json.Schema.Validation" Version="1.0.1" />
1314
<PackageReference Include="CsQuery.Core" Version="2.0.1" />
1415
<PackageReference Include="JsonDiffPatch" Version="2.0.49" />
1516
<PackageReference Include="JsonDiffPatch.Net" Version="2.1.0" />
1617
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
17-
<PackageReference Include="RazorLight.Unofficial" Version="2.0.0-beta1.3" />
1818
<PackageReference Include="ShellProgressBar" Version="5.0.0" />
1919
<PackageReference Include="YamlDotNet" Version="6.0.0" />
2020
</ItemGroup>

‎src/Elastic.CommonSchema.Generator/FileGenerator.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ public static void Generate(CommonSchemaTypesProjection commonSchemaTypesProject
2626
{
2727
{ m => Generate(m, "EcsDocument"), "Base ECS Document" },
2828
{ m => Generate(m, "EcsDocumentJsonConverter"), "Base ECS Document Json Converter" },
29+
{ m => Generate(m, "LogTemplateProperties"), "Strongly types ECS fields supported in log templates" },
30+
{ m => Generate(m, "PropDispatch"), "ECS key value setter generation" },
2931
{ m => Generate(m, "EcsJsonContext"), "Ecs System Text Json Source Generators" },
3032
{ m => Generate(m, "FieldSets"), "Field Sets" },
3133
{ m => Generate(m, "Entities"), "Entities" },
@@ -35,7 +37,7 @@ public static void Generate(CommonSchemaTypesProjection commonSchemaTypesProject
3537
};
3638

3739
using (var progressBar = new ProgressBar(actions.Count, "Generating code",
38-
new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray }))
40+
new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray }))
3941
{
4042
foreach (var kv in actions)
4143
{

‎src/Elastic.CommonSchema.Generator/Projection/ProjectionTypeExtensions.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ public static class ProjectionTypeExtensions
1111
public static string PascalCase(this string s) => new CultureInfo("en-US")
1212
.TextInfo
1313
.ToTitleCase(s.ToLowerInvariant())
14+
.Replace("@", string.Empty)
1415
.Replace("_", string.Empty)
1516
.Replace(".", string.Empty);
1617

@@ -23,6 +24,35 @@ public static string GetClrType(this Field field)
2324
return field.Normalize.Contains("array") ? $"{baseType}[]" : baseType;
2425
}
2526

27+
public static string GetCastFromObject(this Field field)
28+
{
29+
if (field.Normalize.Contains("array")) return null;
30+
switch (field.Type)
31+
{
32+
case FieldType.Keyword:
33+
case FieldType.ConstantKeyword:
34+
case FieldType.Flattened:
35+
case FieldType.MatchOnlyText:
36+
case FieldType.Wildcard:
37+
case FieldType.Text:
38+
case FieldType.Ip:
39+
return "TrySetString";
40+
case FieldType.Boolean:
41+
return "TrySetBool";
42+
case FieldType.ScaledFloat:
43+
case FieldType.Float:
44+
return "TrySetFloat";
45+
case FieldType.Long:
46+
return "TrySetLong";
47+
case FieldType.Integer:
48+
return "TrySetInt";
49+
case FieldType.Date:
50+
return "TrySetDateTimeOffset";
51+
default: return null;
52+
}
53+
54+
}
55+
2656
private static string GetClrType(this FieldType fieldType)
2757
{
2858
switch (fieldType)

‎src/Elastic.CommonSchema.Generator/Projection/PropertyReference.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ protected PropertyReference(string localPath, string fullPath)
1818

1919
public string LocalPath { get; }
2020
public string FullPath { get; }
21+
public string LogTemplateAlternative => FullPath.PascalCase();
2122

2223
public abstract string Description { get; }
2324
public abstract string Example { get; }
@@ -79,11 +80,12 @@ public class ValueTypePropertyReference : PropertyReference
7980
public ValueTypePropertyReference(string parentPath, string fullPath, Field field) : base(parentPath, fullPath)
8081
{
8182
ClrType = field.GetClrType();
83+
CastFromObject = field.GetCastFromObject();
8284
Description = GetFieldDescription(field);
8385
Example = field.Example?.ToString() ?? string.Empty;
84-
8586
}
8687

88+
public string CastFromObject { get; }
8789
public string ClrType { get; }
8890
public override string Description { get; }
8991
public override string Example { get; }

‎src/Elastic.CommonSchema.Generator/Projection/Types.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ public class FieldSetBaseClass
1313

1414
public Dictionary<string, PropertyReference> Properties { get; } = new();
1515

16+
public IEnumerable<ValueTypePropertyReference> SettableProperties =>
17+
ValueProperties.Where(p => !string.IsNullOrEmpty(p.CastFromObject));
18+
1619
public IEnumerable<ValueTypePropertyReference> ValueProperties =>
1720
Properties.Values.OfType<ValueTypePropertyReference>();
1821

‎src/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ namespace Elastic.CommonSchema
3636

3737
/// <summary>
3838
/// Container for additional metadata against this event.
39+
/// <para/>
40+
/// When working with unknown fields use <see cref="SetAnyField"/>. <br/>
41+
/// <para> This will try to assign valid ECS fields to their respective property
42+
/// Failing that it will assign strings to <see cref="Labels"/> and everything else to <see cref="Metadata"/> </para>
3943
/// </summary>
4044
[JsonPropertyName("metadata"), DataMember(Name = "metadata")]
4145
[JsonConverter(typeof(MetadataDictionaryConverter))]

‎src/Elastic.CommonSchema.Generator/Views/LogTemplateProperties.Generated.cshtml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
@using RazorLight
2+
@using System
3+
@using Generator
4+
@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage<Elastic.CommonSchema.Generator.Projection.CommonSchemaTypesProjection>
5+
// Licensed to Elasticsearch B.V under one or more agreements.
6+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
7+
// See the LICENSE file in the project root for more information
8+
9+
/*
10+
IMPORTANT NOTE
11+
==============
12+
This file has been generated.
13+
If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks!
14+
*/
15+
16+
// ReSharper disable RedundantUsingDirective
17+
using System;
18+
using System.Collections.Generic;
19+
using System.Threading;
20+
using System.Threading.Tasks;
21+
using System.Linq;
22+
using System.Net;
23+
using System.Runtime.Serialization;
24+
using System.Text.Json.Serialization;
25+
using Elastic.CommonSchema.Serialization;
26+
using static Elastic.CommonSchema.PropDispatch;
27+
28+
namespace Elastic.CommonSchema
29+
{
30+
public static class LogTemplateProperties
31+
{
32+
@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
33+
{
34+
<text> public static string @prop.LogTemplateAlternative = nameof(@prop.LogTemplateAlternative);
35+
</text>
36+
}
37+
@foreach (var entity in Model.EntityClasses)
38+
{
39+
@foreach (var prop in entity.BaseFieldSet.SettableProperties)
40+
{
41+
<text> public static string @prop.LogTemplateAlternative = nameof(@prop.LogTemplateAlternative);
42+
</text>
43+
44+
}
45+
}
46+
47+
public static readonly HashSet@(Raw("<string>")) All = new()
48+
{
49+
@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
50+
{
51+
<text> "@prop.FullPath", @prop.LogTemplateAlternative,
52+
</text>
53+
54+
}
55+
@foreach (var entity in Model.EntityClasses)
56+
{
57+
@foreach (var prop in entity.BaseFieldSet.SettableProperties)
58+
{
59+
<text> "@prop.FullPath", @prop.LogTemplateAlternative,
60+
</text>
61+
62+
}
63+
}
64+
};
65+
}
66+
67+
}

‎src/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
@using RazorLight
2+
@using System
3+
@using Generator
4+
@using System.Linq;
5+
@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage<Elastic.CommonSchema.Generator.Projection.CommonSchemaTypesProjection>
6+
// Licensed to Elasticsearch B.V under one or more agreements.
7+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
8+
// See the LICENSE file in the project root for more information
9+
10+
/*
11+
IMPORTANT NOTE
12+
==============
13+
This file has been generated.
14+
If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks!
15+
*/
16+
17+
// ReSharper disable RedundantUsingDirective
18+
using System;
19+
using System.Collections.Generic;
20+
using System.Threading;
21+
using System.Threading.Tasks;
22+
using System.Linq;
23+
using System.Net;
24+
using System.Runtime.Serialization;
25+
using System.Text.Json.Serialization;
26+
using Elastic.CommonSchema.Serialization;
27+
using static Elastic.CommonSchema.PropDispatch;
28+
29+
namespace Elastic.CommonSchema
30+
{
31+
///<inheritdoc cref="@Model.Base.BaseFieldSet.Name"/>
32+
public partial class @Model.Base.Name : @Model.Base.BaseFieldSet.Name
33+
{
34+
/// <summary>
35+
/// Set ECS fields by name on <see cref="EcsDocument"/>.
36+
/// <para>Allows valid ECS fields to be set from log message templates.</para>
37+
/// Given <paramref name="value"/>'s type matches the corresponding property on <see cref="EcsDocument"/>
38+
/// <para></para>
39+
/// <para>See <see cref="LogTemplateProperties"/> for a strongly typed list of valid ECS log template properties</para>
40+
/// <para>If its not a supported ECS log template property or using the wrong type:</para>
41+
/// <list type="bullet">
42+
/// <item>Assigns strings to <see cref="EcsDocument.Labels"/> on <see cref="EcsDocument"/></item>
43+
/// <item>Assigns everything else to <see cref="EcsDocument.Metadata"/> on <see cref="EcsDocument"/></item>
44+
/// </list>
45+
/// </summary>
46+
/// <para@(Raw("m")) name="path">Either a supported ECS Log Template property or any key</para@(Raw("m"))>
47+
/// <para@(Raw("m")) name="value">The value to persist</para@(Raw("m"))>
48+
public void SetLogMessageProperty(string path, object value)
49+
{
50+
var assigned = LogTemplateProperties.All.Contains(path) && TrySet(this, path, value);
51+
if (!assigned)
52+
SetMetaOrLabel(this, path, value);
53+
}
54+
}
55+
internal static partial class PropDispatch
56+
{
57+
public static bool TrySet(EcsDocument document, string path, object value)
58+
{
59+
switch (path)
60+
{
61+
@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
62+
{
63+
<text> case "@prop.FullPath":
64+
case "@prop.LogTemplateAlternative":
65+
</text>
66+
}
67+
return TrySet@(@Model.Base.Name)(document, path, value);
68+
@foreach (var entity in Model.EntityClasses)
69+
{
70+
if (!entity.BaseFieldSet.SettableProperties.Any())
71+
{
72+
continue;
73+
}
74+
@foreach (var prop in entity.BaseFieldSet.SettableProperties)
75+
{
76+
<text> case "@prop.FullPath":
77+
case "@prop.LogTemplateAlternative":
78+
</text>
79+
}
80+
<text> return TrySet@(@entity.Name)(document, path, value);
81+
</text>
82+
}
83+
default:
84+
SetMetaOrLabel(document, path, value);
85+
return true;
86+
}
87+
}
88+
89+
public static bool TrySet@(@Model.Base.Name)(EcsDocument document, string path, object value)
90+
{
91+
Func@(Raw("<"))@(Model.Base.Name), object, bool@(Raw(">")) assign = path switch
92+
{
93+
@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
94+
{
95+
<text> "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
96+
"@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
97+
</text>
98+
}
99+
_ => null
100+
};
101+
return assign != null && assign(document, value);
102+
}
103+
@foreach (var entity in Model.EntityClasses)
104+
{
105+
<text>
106+
public static bool TrySet@(entity.Name)(EcsDocument document, string path, object value)
107+
{
108+
Func@(Raw("<"))@(entity.Name), object, bool@(Raw(">")) assign = path switch
109+
{
110+
@foreach (var prop in entity.BaseFieldSet.SettableProperties)
111+
{
112+
<text> "@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
113+
"@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
114+
</text>
115+
}
116+
_ => null
117+
};
118+
if (assign == null) return false;
119+
120+
var entity = document.@(entity.Name) ?? new @(entity.Name)();
121+
var assigned = assign(entity, value);
122+
if (assigned) document.@(entity.Name) = entity;
123+
return assigned;
124+
}
125+
</text>
126+
}
127+
}
128+
}

‎src/Elastic.CommonSchema.Log4net/LoggingEventConverter.cs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ namespace Elastic.CommonSchema.Log4net;
1313
internal static class LoggingEventConverter
1414
{
1515
public static EcsDocument ToEcs(this LoggingEvent loggingEvent)
16-
=> new()
16+
{
17+
var ecsDocument = new EcsDocument()
1718
{
1819
Timestamp = loggingEvent.TimeStamp,
1920
Ecs = new Ecs { Version = EcsDocument.Version },
@@ -23,8 +24,14 @@ public static EcsDocument ToEcs(this LoggingEvent loggingEvent)
2324
Error = GetError(loggingEvent),
2425
Process = GetProcess(loggingEvent),
2526
Host = GetHost(loggingEvent),
26-
Metadata = GetMetadata(loggingEvent)
2727
};
28+
var metadata = GetMetadata(loggingEvent);
29+
if (metadata == null) return ecsDocument;
30+
31+
foreach(var kv in metadata)
32+
ecsDocument.SetLogMessageProperty(kv.Key, kv.Value);
33+
return ecsDocument;
34+
}
2835

2936
private static Log GetLog(LoggingEvent loggingEvent)
3037
{
@@ -134,9 +141,7 @@ private static MetadataDictionary GetMetadata(LoggingEvent loggingEvent)
134141
{
135142
var properties = loggingEvent.GetProperties();
136143
if (properties.Count == 0)
137-
{
138144
return null;
139-
}
140145

141146
var metadata = new MetadataDictionary();
142147

‎src/Elastic.CommonSchema.Log4net/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ The `Layout = new EcsLayout()` line then instructs log4net to use ECS layout.
3636
The sample above uses the console appender, but you are free to use any appender of your choice, perhaps consider using a
3737
filesystem target and [Elastic Filebeat](https://www.elastic.co/downloads/beats/filebeat) for durable and reliable ingestion.
3838

39+
### ECS Aware Properties
40+
41+
Any valid ECS log template properties that is available under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId`
42+
is supported and will directly set the appropriate ECS field.
43+
3944
## Output
4045

4146
Apart from [mandatory fields](https://www.elastic.co/guide/en/ecs/current/ecs-guidelines.html#_general_guidelines), the output contains additional data:

0 commit comments

Comments
 (0)