From 78fa006f6ae50f55248fd252774fa17d75262557 Mon Sep 17 00:00:00 2001 From: Steve Gordon Date: Wed, 24 Feb 2021 12:55:29 +0000 Subject: [PATCH] Add network direction ingest processor --- src/Nest/Ingest/ProcessorFormatter.cs | 9 +- .../Processors/NetworkDirectionProcessor.cs | 87 +++++++++++++++++++ src/Nest/Ingest/ProcessorsDescriptor.cs | 4 + tests/Tests/Ingest/ProcessorAssertions.cs | 35 ++++++++ 4 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 src/Nest/Ingest/Processors/NetworkDirectionProcessor.cs diff --git a/src/Nest/Ingest/ProcessorFormatter.cs b/src/Nest/Ingest/ProcessorFormatter.cs index d4728199e3d..f98d20b1c7d 100644 --- a/src/Nest/Ingest/ProcessorFormatter.cs +++ b/src/Nest/Ingest/ProcessorFormatter.cs @@ -48,7 +48,8 @@ internal class ProcessorFormatter : IJsonFormatter { "csv", 32 }, { "uri_parts", 33 }, { "fingerprint", 34 }, - { "community_id", 35 } + { "community_id", 35 }, + { "network_direction", 36 } }; public IProcessor Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver) @@ -177,6 +178,9 @@ public IProcessor Deserialize(ref JsonReader reader, IJsonFormatterResolver form case 35: processor = Deserialize(ref reader, formatterResolver); break; + case 36: + processor = Deserialize(ref reader, formatterResolver); + break; } } else @@ -306,6 +310,9 @@ public void Serialize(ref JsonWriter writer, IProcessor value, IJsonFormatterRes case "community_id": Serialize(ref writer, value, formatterResolver); break; + case "network_direction": + Serialize(ref writer, value, formatterResolver); + break; default: var formatter = DynamicObjectResolver.ExcludeNullCamelCase.GetFormatter(); formatter.Serialize(ref writer, value, formatterResolver); diff --git a/src/Nest/Ingest/Processors/NetworkDirectionProcessor.cs b/src/Nest/Ingest/Processors/NetworkDirectionProcessor.cs new file mode 100644 index 00000000000..f2336207fd2 --- /dev/null +++ b/src/Nest/Ingest/Processors/NetworkDirectionProcessor.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Runtime.Serialization; +using Elasticsearch.Net.Utf8Json; + +namespace Nest +{ + [InterfaceDataContract] + public interface INetworkDirectionProcessor : IProcessor + { + [DataMember(Name = "destination_ip")] + Field DestinationIp { get; set; } + + [DataMember(Name = "ignore_missing")] + bool? IgnoreMissing { get; set; } + + [DataMember(Name = "internal_networks")] + IEnumerable InternalNetworks { get; set; } + + [DataMember(Name = "source_ip")] + Field SourceIp { get; set; } + + [DataMember(Name = "target_field")] + Field TargetField { get; set; } + } + + public class NetworkDirectionProcessor : ProcessorBase, INetworkDirectionProcessor + { + protected override string Name => "network_direction"; + + /// + public Field DestinationIp { get; set; } + /// + public bool? IgnoreMissing { get; set; } + /// + public IEnumerable InternalNetworks { get; set; } + /// + public Field SourceIp { get; set; } + /// + public Field TargetField { get; set; } + } + + /// + public class NetworkDirectionProcessorDescriptor + : ProcessorDescriptorBase, INetworkDirectionProcessor>, INetworkDirectionProcessor + where T : class + { + protected override string Name => "network_direction"; + + Field INetworkDirectionProcessor.DestinationIp { get; set; } + bool? INetworkDirectionProcessor.IgnoreMissing { get; set; } + IEnumerable INetworkDirectionProcessor.InternalNetworks { get; set; } + Field INetworkDirectionProcessor.SourceIp { get; set; } + Field INetworkDirectionProcessor.TargetField { get; set; } + + /// + public NetworkDirectionProcessorDescriptor DestinationIp(Field field) => Assign(field, (a, v) => a.DestinationIp = v); + + /// + public NetworkDirectionProcessorDescriptor DestinationIp(Expression> objectPath) => + Assign(objectPath, (a, v) => a.DestinationIp = v); + + /// + public NetworkDirectionProcessorDescriptor IgnoreMissing(bool? ignoreMissing = true) => Assign(ignoreMissing, (a, v) => a.IgnoreMissing = v); + + /// + public NetworkDirectionProcessorDescriptor InternalNetworks(IEnumerable internalNetworks) => Assign(internalNetworks, (a, v) => a.InternalNetworks = v); + + /// + public NetworkDirectionProcessorDescriptor InternalNetworks(params string[] internalNetworks) => Assign(internalNetworks, (a, v) => a.InternalNetworks = v); + + /// + public NetworkDirectionProcessorDescriptor SourceIp(Field field) => Assign(field, (a, v) => a.SourceIp = v); + + /// + public NetworkDirectionProcessorDescriptor SourceIp(Expression> objectPath) => + Assign(objectPath, (a, v) => a.SourceIp = v); + + /// + public NetworkDirectionProcessorDescriptor TargetField(Field field) => Assign(field, (a, v) => a.TargetField = v); + + /// + public NetworkDirectionProcessorDescriptor TargetField(Expression> objectPath) => + Assign(objectPath, (a, v) => a.TargetField = v); + } +} diff --git a/src/Nest/Ingest/ProcessorsDescriptor.cs b/src/Nest/Ingest/ProcessorsDescriptor.cs index d18c3836093..ba3180eb4c1 100644 --- a/src/Nest/Ingest/ProcessorsDescriptor.cs +++ b/src/Nest/Ingest/ProcessorsDescriptor.cs @@ -189,5 +189,9 @@ public ProcessorsDescriptor Fingerprint(Func public ProcessorsDescriptor NetworkCommunityId(Func, INetworkCommunityIdProcessor> selector) where T : class => Assign(selector, (a, v) => a.AddIfNotNull(v?.Invoke(new NetworkCommunityIdProcessorDescriptor()))); + + /// + public ProcessorsDescriptor NetworkDirection(Func, INetworkDirectionProcessor> selector) where T : class => + Assign(selector, (a, v) => a.AddIfNotNull(v?.Invoke(new NetworkDirectionProcessorDescriptor()))); } } diff --git a/tests/Tests/Ingest/ProcessorAssertions.cs b/tests/Tests/Ingest/ProcessorAssertions.cs index 3207ca21c26..748291c9e54 100644 --- a/tests/Tests/Ingest/ProcessorAssertions.cs +++ b/tests/Tests/Ingest/ProcessorAssertions.cs @@ -783,5 +783,40 @@ public class NetworkCommunityId : ProcessorAssertion public override string Key => "community_id"; } + + [SkipVersion("<7.12.0", "Uses network direction processor which was introduced in 7.12.0")] + public class NetworkDirection : ProcessorAssertion + { + public override Func>> Fluent => d => d + .NetworkDirection(ud => ud + .DestinationIp(f => f.LeadDeveloper.IpAddress) + .SourceIp(f => f.LeadDeveloper.IpAddress) + .InternalNetworks("network-a", "network-b") + .TargetField(p => p.Description) + .IgnoreMissing()); + + public override IProcessor Initializer => new NetworkDirectionProcessor + { + DestinationIp = Field(f => f.LeadDeveloper.IpAddress), + SourceIp = Field(f => f.LeadDeveloper.IpAddress), + InternalNetworks = new [] { "network-a", "network-b" }, + TargetField = "description", + IgnoreMissing = true + }; + + public override object Json => new + { + destination_ip = "leadDeveloper.ipAddress", + internal_networks = new[] + { + "network-a", "network-b" + }, + source_ip = "leadDeveloper.ipAddress", + target_field = "description", + ignore_missing = true + }; + + public override string Key => "network_direction"; + } } }