From 96377981cca842b0f0e6be2216bfb31389576a48 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Thu, 2 Oct 2025 14:21:35 -0400 Subject: [PATCH 01/31] Good place to checkpoint --- .../common/NetworkDirectionProcessor.java | 100 +-------- .../common/network/NetworkDirectionUtils.java | 108 ++++++++++ .../scalar/ip/NetworkDirectionEvaluator.java | 202 ++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../function/EsqlFunctionRegistry.java | 2 + .../scalar/ScalarFunctionWritables.java | 2 + .../function/scalar/ip/NetworkDirection.java | 165 ++++++++++++++ 7 files changed, 488 insertions(+), 98 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index 7257eaf71a3cb..1d53b89942e46 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -10,8 +10,7 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.cluster.metadata.ProjectId; -import org.elasticsearch.common.network.CIDRUtils; -import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkDirectionUtils; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -19,9 +18,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; -import java.net.InetAddress; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -29,9 +26,6 @@ import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty; public class NetworkDirectionProcessor extends AbstractProcessor { - static final byte[] UNDEFINED_IP4 = new byte[] { 0, 0, 0, 0 }; - static final byte[] UNDEFINED_IP6 = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - static final byte[] BROADCAST_IP4 = new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff }; public static final String TYPE = "network_direction"; @@ -40,17 +34,6 @@ public class NetworkDirectionProcessor extends AbstractProcessor { public static final String DIRECTION_INBOUND = "inbound"; public static final String DIRECTION_OUTBOUND = "outbound"; - private static final String LOOPBACK_NAMED_NETWORK = "loopback"; - private static final String GLOBAL_UNICAST_NAMED_NETWORK = "global_unicast"; - private static final String UNICAST_NAMED_NETWORK = "unicast"; - private static final String LINK_LOCAL_UNICAST_NAMED_NETWORK = "link_local_unicast"; - private static final String INTERFACE_LOCAL_NAMED_NETWORK = "interface_local_multicast"; - private static final String LINK_LOCAL_MULTICAST_NAMED_NETWORK = "link_local_multicast"; - private static final String MULTICAST_NAMED_NETWORK = "multicast"; - private static final String UNSPECIFIED_NAMED_NETWORK = "unspecified"; - private static final String PRIVATE_NAMED_NETWORK = "private"; - private static final String PUBLIC_NAMED_NETWORK = "public"; - private final String sourceIpField; private final String destinationIpField; private final String targetField; @@ -140,8 +123,8 @@ private String getDirection(IngestDocument d) throws Exception { return null; } - boolean sourceInternal = isInternal(networks, sourceIpAddrString); - boolean destinationInternal = isInternal(networks, destIpAddrString); + boolean sourceInternal = NetworkDirectionUtils.isInternal(networks, sourceIpAddrString); + boolean destinationInternal = NetworkDirectionUtils.isInternal(networks, destIpAddrString); if (sourceInternal && destinationInternal) { return DIRECTION_INTERNAL; @@ -155,83 +138,6 @@ private String getDirection(IngestDocument d) throws Exception { return DIRECTION_EXTERNAL; } - private static boolean isInternal(List networks, String ip) { - for (String network : networks) { - if (inNetwork(ip, network)) { - return true; - } - } - return false; - } - - private static boolean inNetwork(String ip, String network) { - InetAddress address = InetAddresses.forString(ip); - return switch (network) { - case LOOPBACK_NAMED_NETWORK -> isLoopback(address); - case GLOBAL_UNICAST_NAMED_NETWORK, UNICAST_NAMED_NETWORK -> isUnicast(address); - case LINK_LOCAL_UNICAST_NAMED_NETWORK -> isLinkLocalUnicast(address); - case INTERFACE_LOCAL_NAMED_NETWORK -> isInterfaceLocalMulticast(address); - case LINK_LOCAL_MULTICAST_NAMED_NETWORK -> isLinkLocalMulticast(address); - case MULTICAST_NAMED_NETWORK -> isMulticast(address); - case UNSPECIFIED_NAMED_NETWORK -> isUnspecified(address); - case PRIVATE_NAMED_NETWORK -> isPrivate(ip); - case PUBLIC_NAMED_NETWORK -> isPublic(ip); - default -> CIDRUtils.isInRange(ip, network); - }; - } - - private static boolean isLoopback(InetAddress ip) { - return ip.isLoopbackAddress(); - } - - private static boolean isUnicast(InetAddress ip) { - return Arrays.equals(ip.getAddress(), BROADCAST_IP4) == false - && isUnspecified(ip) == false - && isLoopback(ip) == false - && isMulticast(ip) == false - && isLinkLocalUnicast(ip) == false; - } - - private static boolean isLinkLocalUnicast(InetAddress ip) { - return ip.isLinkLocalAddress(); - } - - private static boolean isInterfaceLocalMulticast(InetAddress ip) { - return ip.isMCNodeLocal(); - } - - private static boolean isLinkLocalMulticast(InetAddress ip) { - return ip.isMCLinkLocal(); - } - - private static boolean isMulticast(InetAddress ip) { - return ip.isMulticastAddress(); - } - - private static boolean isUnspecified(InetAddress ip) { - var address = ip.getAddress(); - return Arrays.equals(UNDEFINED_IP4, address) || Arrays.equals(UNDEFINED_IP6, address); - } - - private static boolean isPrivate(String ip) { - return CIDRUtils.isInRange(ip, "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"); - } - - private static boolean isPublic(String ip) { - return isLocalOrPrivate(ip) == false; - } - - private static boolean isLocalOrPrivate(String ip) { - var address = InetAddresses.forString(ip); - return isPrivate(ip) - || isLoopback(address) - || isUnspecified(address) - || isLinkLocalUnicast(address) - || isLinkLocalMulticast(address) - || isInterfaceLocalMulticast(address) - || Arrays.equals(address.getAddress(), BROADCAST_IP4); - } - @Override public String getType() { return TYPE; diff --git a/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java b/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java new file mode 100644 index 0000000000000..9c4eb3d410a33 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.common.network; + +import java.net.InetAddress; +import java.util.Arrays; +import java.util.List; + +public class NetworkDirectionUtils { + + static final byte[] UNDEFINED_IP4 = new byte[] { 0, 0, 0, 0 }; + static final byte[] UNDEFINED_IP6 = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + static final byte[] BROADCAST_IP4 = new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff }; + + private static final String LOOPBACK_NAMED_NETWORK = "loopback"; + private static final String GLOBAL_UNICAST_NAMED_NETWORK = "global_unicast"; + private static final String UNICAST_NAMED_NETWORK = "unicast"; + private static final String LINK_LOCAL_UNICAST_NAMED_NETWORK = "link_local_unicast"; + private static final String INTERFACE_LOCAL_NAMED_NETWORK = "interface_local_multicast"; + private static final String LINK_LOCAL_MULTICAST_NAMED_NETWORK = "link_local_multicast"; + private static final String MULTICAST_NAMED_NETWORK = "multicast"; + private static final String UNSPECIFIED_NAMED_NETWORK = "unspecified"; + private static final String PRIVATE_NAMED_NETWORK = "private"; + private static final String PUBLIC_NAMED_NETWORK = "public"; + + public static boolean isInternal(List networks, String ip) { + for (String network : networks) { + if (inNetwork(InetAddresses.forString(ip), network)) { + return true; + } + } + return false; + } + + public static boolean inNetwork(InetAddress address, String network) { + return switch (network) { + case LOOPBACK_NAMED_NETWORK -> isLoopback(address); + case GLOBAL_UNICAST_NAMED_NETWORK, UNICAST_NAMED_NETWORK -> isUnicast(address); + case LINK_LOCAL_UNICAST_NAMED_NETWORK -> isLinkLocalUnicast(address); + case INTERFACE_LOCAL_NAMED_NETWORK -> isInterfaceLocalMulticast(address); + case LINK_LOCAL_MULTICAST_NAMED_NETWORK -> isLinkLocalMulticast(address); + case MULTICAST_NAMED_NETWORK -> isMulticast(address); + case UNSPECIFIED_NAMED_NETWORK -> isUnspecified(address); + case PRIVATE_NAMED_NETWORK -> isPrivate(address.getHostAddress()); + case PUBLIC_NAMED_NETWORK -> isPublic(address.getHostAddress()); + default -> CIDRUtils.isInRange(address.getHostAddress(), network); + }; + } + + private static boolean isLoopback(InetAddress ip) { + return ip.isLoopbackAddress(); + } + + private static boolean isUnicast(InetAddress ip) { + return Arrays.equals(ip.getAddress(), BROADCAST_IP4) == false + && isUnspecified(ip) == false + && isLoopback(ip) == false + && isMulticast(ip) == false + && isLinkLocalUnicast(ip) == false; + } + + private static boolean isLinkLocalUnicast(InetAddress ip) { + return ip.isLinkLocalAddress(); + } + + private static boolean isInterfaceLocalMulticast(InetAddress ip) { + return ip.isMCNodeLocal(); + } + + private static boolean isLinkLocalMulticast(InetAddress ip) { + return ip.isMCLinkLocal(); + } + + private static boolean isMulticast(InetAddress ip) { + return ip.isMulticastAddress(); + } + + private static boolean isUnspecified(InetAddress ip) { + var address = ip.getAddress(); + return Arrays.equals(UNDEFINED_IP4, address) || Arrays.equals(UNDEFINED_IP6, address); + } + + private static boolean isPrivate(String ip) { + return CIDRUtils.isInRange(ip, "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"); + } + + private static boolean isPublic(String ip) { + return isLocalOrPrivate(ip) == false; + } + + private static boolean isLocalOrPrivate(String ip) { + var address = InetAddresses.forString(ip); + return isPrivate(ip) + || isLoopback(address) + || isUnspecified(address) + || isLinkLocalUnicast(address) + || isLinkLocalMulticast(address) + || isInterfaceLocalMulticast(address) + || Arrays.equals(address.getAddress(), BROADCAST_IP4); + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java new file mode 100644 index 0000000000000..394e01fa7964a --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.util.function.Function; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link NetworkDirection}. + * This class is generated. Edit {@code EvaluatorImplementer} instead. + */ +public final class NetworkDirectionEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(NetworkDirectionEvaluator.class); + + private final Source source; + + private final EvalOperator.ExpressionEvaluator sourceIp; + + private final EvalOperator.ExpressionEvaluator destinationIp; + + private final EvalOperator.ExpressionEvaluator internalNetworks; + + private final BreakingBytesRefBuilder scratch; + + private final DriverContext driverContext; + + private Warnings warnings; + + public NetworkDirectionEvaluator(Source source, EvalOperator.ExpressionEvaluator sourceIp, + EvalOperator.ExpressionEvaluator destinationIp, + EvalOperator.ExpressionEvaluator internalNetworks, BreakingBytesRefBuilder scratch, + DriverContext driverContext) { + this.source = source; + this.sourceIp = sourceIp; + this.destinationIp = destinationIp; + this.internalNetworks = internalNetworks; + this.scratch = scratch; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock sourceIpBlock = (BytesRefBlock) sourceIp.eval(page)) { + try (BytesRefBlock destinationIpBlock = (BytesRefBlock) destinationIp.eval(page)) { + try (BytesRefBlock internalNetworksBlock = (BytesRefBlock) internalNetworks.eval(page)) { + BytesRefVector sourceIpVector = sourceIpBlock.asVector(); + if (sourceIpVector == null) { + return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, internalNetworksBlock); + } + BytesRefVector destinationIpVector = destinationIpBlock.asVector(); + if (destinationIpVector == null) { + return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, internalNetworksBlock); + } + return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, internalNetworksVector).asBlock(); + } + } + } + } + + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += sourceIp.baseRamBytesUsed(); + baseRamBytesUsed += destinationIp.baseRamBytesUsed(); + baseRamBytesUsed += internalNetworks.baseRamBytesUsed(); + return baseRamBytesUsed; + } + + public BooleanBlock eval(int positionCount, BytesRefBlock sourceIpBlock, + BytesRefBlock destinationIpBlock, BytesRefBlock internalNetworksBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + BytesRef sourceIpScratch = new BytesRef(); + BytesRef destinationIpScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (sourceIpBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (sourceIpBlock.getValueCount(p) != 1) { + if (sourceIpBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (destinationIpBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (destinationIpBlock.getValueCount(p) != 1) { + if (destinationIpBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (internalNetworksBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (internalNetworksBlock.getValueCount(p) != 1) { + if (internalNetworksBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + BytesRef sourceIp = sourceIpBlock.getBytesRef(sourceIpBlock.getFirstValueIndex(p), sourceIpScratch); + BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); + result.appendBoolean(NetworkDirection.process(sourceIp, destinationIp, p, internalNetworksBlock, this.scratch)); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, BytesRefVector sourceIpVector, + BytesRefVector destinationIpVector, BytesRefBlock internalNetworksVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + BytesRef sourceIpScratch = new BytesRef(); + BytesRef destinationIpScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + BytesRef sourceIp = sourceIpVector.getBytesRef(p, sourceIpScratch); + BytesRef destinationIp = destinationIpVector.getBytesRef(p, destinationIpScratch); + result.appendBoolean(p, NetworkDirection.process(sourceIp, destinationIp, p, internalNetworksVector, this.scratch)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "NetworkDirectionEvaluator[" + "sourceIp=" + sourceIp + ", destinationIp=" + destinationIp + ", internalNetworks=" + internalNetworks + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(sourceIp, destinationIp, internalNetworks, scratch); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory sourceIp; + + private final EvalOperator.ExpressionEvaluator.Factory destinationIp; + + private final EvalOperator.ExpressionEvaluator.Factory internalNetworks; + + private final Function scratch; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory sourceIp, + EvalOperator.ExpressionEvaluator.Factory destinationIp, + EvalOperator.ExpressionEvaluator.Factory internalNetworks, + Function scratch) { + this.source = source; + this.sourceIp = sourceIp; + this.destinationIp = destinationIp; + this.internalNetworks = internalNetworks; + this.scratch = scratch; + } + + @Override + public NetworkDirectionEvaluator get(DriverContext context) { + return new NetworkDirectionEvaluator(source, sourceIp.get(context), destinationIp.get(context), internalNetworks.get(context), scratch.apply(context), context); + } + + @Override + public String toString() { + return "NetworkDirectionEvaluator[" + "sourceIp=" + sourceIp + ", destinationIp=" + destinationIp + ", internalNetworks=" + internalNetworks + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 818af0aa49ebe..356ec478dd57f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1570,7 +1570,12 @@ public enum Cap { /** * Support for requesting the "_tsid" metadata field. */ - METADATA_TSID_FIELD; + METADATA_TSID_FIELD, + + /** + * Network direction function. + */ + NETWORK_DIRECTION(Build.current().isSnapshot()); private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 45e7849b6a603..96f228db00645 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -109,6 +109,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; +import org.elasticsearch.xpack.esql.expression.function.scalar.ip.NetworkDirection; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Acos; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Asin; @@ -454,6 +455,7 @@ private static FunctionDefinition[][] functions() { // IP new FunctionDefinition[] { def(CIDRMatch.class, CIDRMatch::new, "cidr_match") }, new FunctionDefinition[] { def(IpPrefix.class, IpPrefix::new, "ip_prefix") }, + new FunctionDefinition[] { def(NetworkDirection.class, NetworkDirection::new, "network_direction") }, // conversion functions new FunctionDefinition[] { def(FromBase64.class, FromBase64::new, "from_base64"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index 978f0218a798f..708f027db1b0e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; +import org.elasticsearch.xpack.esql.expression.function.scalar.ip.NetworkDirection; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Atan2; import org.elasticsearch.xpack.esql.expression.function.scalar.math.CopySign; import org.elasticsearch.xpack.esql.expression.function.scalar.math.E; @@ -94,6 +95,7 @@ public static List getNamedWriteables() { entries.add(Locate.ENTRY); entries.add(Log.ENTRY); entries.add(Md5.ENTRY); + entries.add(NetworkDirection.ENTRY); entries.add(Now.ENTRY); entries.add(Or.ENTRY); entries.add(Pi.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java new file mode 100644 index 0000000000000..a3c68f5ea032e --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkDirectionUtils; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.ann.Fixed; +import org.elasticsearch.compute.ann.Position; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL; + +/** + * Returns whether a connection is inbound given a source IP address, destination IP address, and a list of internal networks. + */ +public class NetworkDirection extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "NetworkDirection", + NetworkDirection::new + ); + + private final Expression sourceIpField; + private final Expression destinationIpField; + private final Expression internalNetworks; + + @FunctionInfo( + returnType = "boolean", + description = "Returns true if the direction of the source-to-destination-IPs is determined to be inbound, provided a list of internal networks.", + examples = @Example(file = "ip", tag = "cdirMatchMultipleArgs") // TODO make an example + ) + public NetworkDirection( + Source source, + @Param( + name = "source_ip", + type = { "ip" }, + description = "Source IP address of type `ip` (both IPv4 and IPv6 are supported)." + ) Expression sourceIpField, + @Param( + name = "destination_ip", + type = { "ip" }, + description = "Destination IP address of type `ip` (both IPv4 and IPv6 are supported)." + ) Expression destinationIpField, + @Param( + name = "internal_networks", + type = { "keyword", "text" }, + description = "List of internal networks. Supports IPv4 and IPv6 addresses, ranges in CIDR notation, and named ranges.") + Expression internalNetworks + ) { + super(source, Arrays.asList(sourceIpField, destinationIpField, internalNetworks)); + this.sourceIpField = sourceIpField; + this.destinationIpField = destinationIpField; + this.internalNetworks = internalNetworks; + } + + private NetworkDirection(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + in.readNamedWriteable(Expression.class), + in.readNamedWriteable(Expression.class) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(sourceIpField); + out.writeNamedWriteable(destinationIpField); + out.writeNamedWriteable(internalNetworks); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new NetworkDirection(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, NetworkDirection::new, sourceIpField, destinationIpField, internalNetworks); + } + + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var sourceIpEvaluatorSupplier = toEvaluator.apply(sourceIpField); + var destinationIpEvaluatorSupplier = toEvaluator.apply(destinationIpField); + var internalNetworksEvaluatorSupplier = toEvaluator.apply(internalNetworks); + return new NetworkDirectionEvaluator.Factory( + source(), + sourceIpEvaluatorSupplier, + destinationIpEvaluatorSupplier, + internalNetworksEvaluatorSupplier, + context -> new BreakingBytesRefBuilder(context.breaker(), "network_direction") + ); + } + + @Evaluator + static boolean process(BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock internalNetworks, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BreakingBytesRefBuilder scratch) { + // Pulling the bytes out directly using InetAddress.getByAddress() requires error handling TODO + InetAddress sourceIpAddress = InetAddresses.forString(sourceIp.utf8ToString()); + InetAddress destinationIpAddress = InetAddresses.forString(destinationIp.utf8ToString()); + boolean sourceInternal = false; + boolean destinationInternal = false; + + int valueCount = internalNetworks.getValueCount(position); + int first = internalNetworks.getFirstValueIndex(position); + + for (int i = first; i < first + valueCount; i++) { + if (NetworkDirectionUtils.inNetwork(sourceIpAddress, internalNetworks.getBytesRef(i, scratch).utf8ToString())) { + sourceInternal = true; + break; + } + } + for (int i = first; i < first + valueCount; i++) { + if (NetworkDirectionUtils.inNetwork(destinationIpAddress, internalNetworks.getBytesRef(i, scratch).utf8ToString())) { + destinationInternal = true; + break; + } + } + + if (sourceInternal == false) { + return destinationInternal; + } + return false; + } + + @Override + public DataType dataType() { + return DataType.BOOLEAN; + } +} From 3129e5761911e44d25bfaa182b7b8da12fe2a63a Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Thu, 2 Oct 2025 15:05:15 -0400 Subject: [PATCH 02/31] Fixed codegen bug, completed first pass of Network Direction ESQL function (no tests) --- .../compute/gen/argument/BlockArgument.java | 2 +- .../scalar/ip/NetworkDirectionEvaluator.java | 38 +++++++++---------- .../function/scalar/ip/NetworkDirection.java | 9 ++--- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/BlockArgument.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/BlockArgument.java index 42f0a1eaf04b4..8adbd7a40c2d2 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/BlockArgument.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/BlockArgument.java @@ -26,7 +26,7 @@ public TypeName dataType(boolean blockStyle) { @Override public String paramName(boolean blockStyle) { - return name + (blockStyle ? "Block" : "Vector"); + return name + "Block"; } @Override diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index 394e01fa7964a..7fb89cc709a1b 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -16,7 +16,6 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Warnings; @@ -32,27 +31,26 @@ public final class NetworkDirectionEvaluator implements EvalOperator.ExpressionE private final Source source; + private final BytesRef scratch; + private final EvalOperator.ExpressionEvaluator sourceIp; private final EvalOperator.ExpressionEvaluator destinationIp; private final EvalOperator.ExpressionEvaluator internalNetworks; - private final BreakingBytesRefBuilder scratch; - private final DriverContext driverContext; private Warnings warnings; - public NetworkDirectionEvaluator(Source source, EvalOperator.ExpressionEvaluator sourceIp, - EvalOperator.ExpressionEvaluator destinationIp, - EvalOperator.ExpressionEvaluator internalNetworks, BreakingBytesRefBuilder scratch, - DriverContext driverContext) { + public NetworkDirectionEvaluator(Source source, BytesRef scratch, + EvalOperator.ExpressionEvaluator sourceIp, EvalOperator.ExpressionEvaluator destinationIp, + EvalOperator.ExpressionEvaluator internalNetworks, DriverContext driverContext) { this.source = source; + this.scratch = scratch; this.sourceIp = sourceIp; this.destinationIp = destinationIp; this.internalNetworks = internalNetworks; - this.scratch = scratch; this.driverContext = driverContext; } @@ -69,7 +67,7 @@ public Block eval(Page page) { if (destinationIpVector == null) { return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, internalNetworksBlock); } - return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, internalNetworksVector).asBlock(); + return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, internalNetworksBlock).asBlock(); } } } @@ -125,21 +123,21 @@ public BooleanBlock eval(int positionCount, BytesRefBlock sourceIpBlock, } BytesRef sourceIp = sourceIpBlock.getBytesRef(sourceIpBlock.getFirstValueIndex(p), sourceIpScratch); BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); - result.appendBoolean(NetworkDirection.process(sourceIp, destinationIp, p, internalNetworksBlock, this.scratch)); + result.appendBoolean(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); } return result.build(); } } public BooleanVector eval(int positionCount, BytesRefVector sourceIpVector, - BytesRefVector destinationIpVector, BytesRefBlock internalNetworksVector) { + BytesRefVector destinationIpVector, BytesRefBlock internalNetworksBlock) { try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { BytesRef sourceIpScratch = new BytesRef(); BytesRef destinationIpScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { BytesRef sourceIp = sourceIpVector.getBytesRef(p, sourceIpScratch); BytesRef destinationIp = destinationIpVector.getBytesRef(p, destinationIpScratch); - result.appendBoolean(p, NetworkDirection.process(sourceIp, destinationIp, p, internalNetworksVector, this.scratch)); + result.appendBoolean(p, NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); } return result.build(); } @@ -152,7 +150,7 @@ public String toString() { @Override public void close() { - Releasables.closeExpectNoException(sourceIp, destinationIp, internalNetworks, scratch); + Releasables.closeExpectNoException(sourceIp, destinationIp, internalNetworks); } private Warnings warnings() { @@ -170,28 +168,28 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; + private final Function scratch; + private final EvalOperator.ExpressionEvaluator.Factory sourceIp; private final EvalOperator.ExpressionEvaluator.Factory destinationIp; private final EvalOperator.ExpressionEvaluator.Factory internalNetworks; - private final Function scratch; - - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory sourceIp, + public Factory(Source source, Function scratch, + EvalOperator.ExpressionEvaluator.Factory sourceIp, EvalOperator.ExpressionEvaluator.Factory destinationIp, - EvalOperator.ExpressionEvaluator.Factory internalNetworks, - Function scratch) { + EvalOperator.ExpressionEvaluator.Factory internalNetworks) { this.source = source; + this.scratch = scratch; this.sourceIp = sourceIp; this.destinationIp = destinationIp; this.internalNetworks = internalNetworks; - this.scratch = scratch; } @Override public NetworkDirectionEvaluator get(DriverContext context) { - return new NetworkDirectionEvaluator(source, sourceIp.get(context), destinationIp.get(context), internalNetworks.get(context), scratch.apply(context), context); + return new NetworkDirectionEvaluator(source, scratch.apply(context), sourceIp.get(context), destinationIp.get(context), internalNetworks.get(context), context); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index a3c68f5ea032e..9086777498ab0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.ann.Position; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -31,10 +30,8 @@ import java.io.IOException; import java.net.InetAddress; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL; @@ -121,15 +118,15 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua var internalNetworksEvaluatorSupplier = toEvaluator.apply(internalNetworks); return new NetworkDirectionEvaluator.Factory( source(), + context -> new BytesRef(), sourceIpEvaluatorSupplier, destinationIpEvaluatorSupplier, - internalNetworksEvaluatorSupplier, - context -> new BreakingBytesRefBuilder(context.breaker(), "network_direction") + internalNetworksEvaluatorSupplier ); } @Evaluator - static boolean process(BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock internalNetworks, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BreakingBytesRefBuilder scratch) { + static boolean process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock internalNetworks) { // Pulling the bytes out directly using InetAddress.getByAddress() requires error handling TODO InetAddress sourceIpAddress = InetAddresses.forString(sourceIp.utf8ToString()); InetAddress destinationIpAddress = InetAddresses.forString(destinationIp.utf8ToString()); From 13fe69db8707acea8e09076f90926a73c4ccfd52 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Sat, 4 Oct 2025 13:25:24 -0400 Subject: [PATCH 03/31] Further refactoring back to keyword return type --- .../common/NetworkDirectionProcessor.java | 16 +------------- .../common/network/NetworkDirectionUtils.java | 18 ++++++++++++++++ .../scalar/ip/NetworkDirectionEvaluator.java | 14 ++++++------- .../function/scalar/ip/NetworkDirection.java | 11 ++++------ .../scalar/ip/NetworkDirectionTests.java | 21 +++++++++++++++++++ 5 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index 1d53b89942e46..19cb4cf642a24 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -29,11 +29,6 @@ public class NetworkDirectionProcessor extends AbstractProcessor { public static final String TYPE = "network_direction"; - public static final String DIRECTION_INTERNAL = "internal"; - public static final String DIRECTION_EXTERNAL = "external"; - public static final String DIRECTION_INBOUND = "inbound"; - public static final String DIRECTION_OUTBOUND = "outbound"; - private final String sourceIpField; private final String destinationIpField; private final String targetField; @@ -126,16 +121,7 @@ private String getDirection(IngestDocument d) throws Exception { boolean sourceInternal = NetworkDirectionUtils.isInternal(networks, sourceIpAddrString); boolean destinationInternal = NetworkDirectionUtils.isInternal(networks, destIpAddrString); - if (sourceInternal && destinationInternal) { - return DIRECTION_INTERNAL; - } - if (sourceInternal) { - return DIRECTION_OUTBOUND; - } - if (destinationInternal) { - return DIRECTION_INBOUND; - } - return DIRECTION_EXTERNAL; + return NetworkDirectionUtils.getDirection(sourceInternal, destinationInternal); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java b/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java index 9c4eb3d410a33..8aaba801daf0a 100644 --- a/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java +++ b/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java @@ -30,6 +30,11 @@ public class NetworkDirectionUtils { private static final String PRIVATE_NAMED_NETWORK = "private"; private static final String PUBLIC_NAMED_NETWORK = "public"; + public static final String DIRECTION_INTERNAL = "internal"; + public static final String DIRECTION_EXTERNAL = "external"; + public static final String DIRECTION_INBOUND = "inbound"; + public static final String DIRECTION_OUTBOUND = "outbound"; + public static boolean isInternal(List networks, String ip) { for (String network : networks) { if (inNetwork(InetAddresses.forString(ip), network)) { @@ -54,6 +59,19 @@ public static boolean inNetwork(InetAddress address, String network) { }; } + public static String getDirection(boolean sourceInternal, boolean destinationInternal) { + if (sourceInternal && destinationInternal) { + return DIRECTION_INTERNAL; + } + if (sourceInternal) { + return DIRECTION_OUTBOUND; + } + if (destinationInternal) { + return DIRECTION_INBOUND; + } + return DIRECTION_EXTERNAL; + } + private static boolean isLoopback(InetAddress ip) { return ip.isLoopbackAddress(); } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index 7fb89cc709a1b..7518f16fb0901 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -11,8 +11,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.BooleanVector; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.Page; @@ -82,9 +80,9 @@ public long baseRamBytesUsed() { return baseRamBytesUsed; } - public BooleanBlock eval(int positionCount, BytesRefBlock sourceIpBlock, + public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, BytesRefBlock destinationIpBlock, BytesRefBlock internalNetworksBlock) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { BytesRef sourceIpScratch = new BytesRef(); BytesRef destinationIpScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { @@ -123,21 +121,21 @@ public BooleanBlock eval(int positionCount, BytesRefBlock sourceIpBlock, } BytesRef sourceIp = sourceIpBlock.getBytesRef(sourceIpBlock.getFirstValueIndex(p), sourceIpScratch); BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); - result.appendBoolean(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); + result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); } return result.build(); } } - public BooleanVector eval(int positionCount, BytesRefVector sourceIpVector, + public BytesRefVector eval(int positionCount, BytesRefVector sourceIpVector, BytesRefVector destinationIpVector, BytesRefBlock internalNetworksBlock) { - try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { BytesRef sourceIpScratch = new BytesRef(); BytesRef destinationIpScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { BytesRef sourceIp = sourceIpVector.getBytesRef(p, sourceIpScratch); BytesRef destinationIp = destinationIpVector.getBytesRef(p, destinationIpScratch); - result.appendBoolean(p, NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); + result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index 9086777498ab0..72a87ae3f9703 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -50,7 +50,7 @@ public class NetworkDirection extends EsqlScalarFunction { private final Expression internalNetworks; @FunctionInfo( - returnType = "boolean", + returnType = "keyword", description = "Returns true if the direction of the source-to-destination-IPs is determined to be inbound, provided a list of internal networks.", examples = @Example(file = "ip", tag = "cdirMatchMultipleArgs") // TODO make an example ) @@ -126,7 +126,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua } @Evaluator - static boolean process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock internalNetworks) { + static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock internalNetworks) { // Pulling the bytes out directly using InetAddress.getByAddress() requires error handling TODO InetAddress sourceIpAddress = InetAddresses.forString(sourceIp.utf8ToString()); InetAddress destinationIpAddress = InetAddresses.forString(destinationIp.utf8ToString()); @@ -149,14 +149,11 @@ static boolean process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) Bytes } } - if (sourceInternal == false) { - return destinationInternal; - } - return false; + return new BytesRef(NetworkDirectionUtils.getDirection(sourceInternal, destinationInternal)); } @Override public DataType dataType() { - return DataType.BOOLEAN; + return DataType.KEYWORD; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java new file mode 100644 index 0000000000000..7bfeb4a983082 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; + +import java.util.List; + +public class NetworkDirectionTests extends AbstractScalarFunctionTestCase { + @Override + protected Expression build(Source source, List args) { + return null; + } +} From 07b6c8fc632826d06e0aa32b1a84ad7ac83cb0a0 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Mon, 6 Oct 2025 10:47:38 -0400 Subject: [PATCH 04/31] Duplicate tests from NetworkDirectionProcessor into NetworkDirectionutils tests --- .../network/NetworkDirectionUtilsTests.java | 48 +++++++++++++++++++ .../function/scalar/ip/NetworkDirection.java | 10 ++-- .../scalar/ip/NetworkDirectionTests.java | 17 ++++++- 3 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java diff --git a/server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java b/server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java new file mode 100644 index 0000000000000..4b2c9f1381326 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.common.network; + +import org.elasticsearch.test.ESTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class NetworkDirectionUtilsTests extends ESTestCase { + public void testCIDR() { + testNetworkDirectionUtils("10.0.1.1", "192.168.1.2", List.of("10.0.0.0/8"), "outbound"); + testNetworkDirectionUtils("192.168.1.2", "10.0.1.1", List.of("10.0.0.0/8"), "inbound"); + } + + public void testUnspecified() { + testNetworkDirectionUtils("0.0.0.0", "0.0.0.0", List.of("unspecified"), "internal"); + testNetworkDirectionUtils("::", "::", List.of("unspecified"), "internal"); + } + + public void testNetworkPrivate() { + testNetworkDirectionUtils("192.168.1.1", "192.168.1.2", List.of("private"), "internal"); + testNetworkDirectionUtils("10.0.1.1", "192.168.1.2", List.of("private"), "internal"); + testNetworkDirectionUtils("192.168.1.1", "172.16.0.1", List.of("private"), "internal"); + testNetworkDirectionUtils("192.168.1.1", "fd12:3456:789a:1::1", List.of("private"), "internal"); + } + + public void testNetworkPublic() { + testNetworkDirectionUtils("192.168.1.1", "192.168.1.2", List.of("public"), "external"); + testNetworkDirectionUtils("10.0.1.1", "192.168.1.2", List.of("public"), "external"); + testNetworkDirectionUtils("192.168.1.1", "172.16.0.1", List.of("public"), "external"); + testNetworkDirectionUtils("192.168.1.1", "fd12:3456:789a:1::1", List.of("public"), "external"); + } + + private void testNetworkDirectionUtils(String source, String destination, List networks, String expectedDirection) { + boolean sourceInternal = NetworkDirectionUtils.isInternal(networks, source); + boolean destinationInternal = NetworkDirectionUtils.isInternal(networks, destination); + assertThat(expectedDirection, equalTo(NetworkDirectionUtils.getDirection(sourceInternal, destinationInternal))); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index 72a87ae3f9703..e80d00db04e40 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -126,24 +126,24 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua } @Evaluator - static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock internalNetworks) { + static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock networks) { // Pulling the bytes out directly using InetAddress.getByAddress() requires error handling TODO InetAddress sourceIpAddress = InetAddresses.forString(sourceIp.utf8ToString()); InetAddress destinationIpAddress = InetAddresses.forString(destinationIp.utf8ToString()); boolean sourceInternal = false; boolean destinationInternal = false; - int valueCount = internalNetworks.getValueCount(position); - int first = internalNetworks.getFirstValueIndex(position); + int valueCount = networks.getValueCount(position); + int first = networks.getFirstValueIndex(position); for (int i = first; i < first + valueCount; i++) { - if (NetworkDirectionUtils.inNetwork(sourceIpAddress, internalNetworks.getBytesRef(i, scratch).utf8ToString())) { + if (NetworkDirectionUtils.inNetwork(sourceIpAddress, networks.getBytesRef(i, scratch).utf8ToString())) { sourceInternal = true; break; } } for (int i = first; i < first + valueCount; i++) { - if (NetworkDirectionUtils.inNetwork(destinationIpAddress, internalNetworks.getBytesRef(i, scratch).utf8ToString())) { + if (NetworkDirectionUtils.inNetwork(destinationIpAddress, networks.getBytesRef(i, scratch).utf8ToString())) { destinationInternal = true; break; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 7bfeb4a983082..8d69746e6ea44 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -7,15 +7,30 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.ip; +import com.carrotsearch.randomizedtesting.annotations.Name; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.NetworkDirectionUtils; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import java.util.List; +import java.util.function.Supplier; public class NetworkDirectionTests extends AbstractScalarFunctionTestCase { + public NetworkDirectionTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + @Override protected Expression build(Source source, List args) { - return null; + return new NetworkDirection(source, args.get(0), args.get(1), args.get(2)); } } From 319552efa7d5e4894527802e3408e9072d1c2d2e Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Mon, 6 Oct 2025 13:20:10 -0400 Subject: [PATCH 05/31] Change how extraction of IP from BytesRef is performed --- .../scalar/ip/NetworkDirectionEvaluator.java | 44 +++++++++---------- .../function/scalar/ip/NetworkDirection.java | 12 +++-- .../scalar/ip/NetworkDirectionTests.java | 23 ++++++++++ 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index 7518f16fb0901..d5be28bf7e017 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -35,7 +35,7 @@ public final class NetworkDirectionEvaluator implements EvalOperator.ExpressionE private final EvalOperator.ExpressionEvaluator destinationIp; - private final EvalOperator.ExpressionEvaluator internalNetworks; + private final EvalOperator.ExpressionEvaluator networks; private final DriverContext driverContext; @@ -43,12 +43,12 @@ public final class NetworkDirectionEvaluator implements EvalOperator.ExpressionE public NetworkDirectionEvaluator(Source source, BytesRef scratch, EvalOperator.ExpressionEvaluator sourceIp, EvalOperator.ExpressionEvaluator destinationIp, - EvalOperator.ExpressionEvaluator internalNetworks, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator networks, DriverContext driverContext) { this.source = source; this.scratch = scratch; this.sourceIp = sourceIp; this.destinationIp = destinationIp; - this.internalNetworks = internalNetworks; + this.networks = networks; this.driverContext = driverContext; } @@ -56,16 +56,16 @@ public NetworkDirectionEvaluator(Source source, BytesRef scratch, public Block eval(Page page) { try (BytesRefBlock sourceIpBlock = (BytesRefBlock) sourceIp.eval(page)) { try (BytesRefBlock destinationIpBlock = (BytesRefBlock) destinationIp.eval(page)) { - try (BytesRefBlock internalNetworksBlock = (BytesRefBlock) internalNetworks.eval(page)) { + try (BytesRefBlock networksBlock = (BytesRefBlock) networks.eval(page)) { BytesRefVector sourceIpVector = sourceIpBlock.asVector(); if (sourceIpVector == null) { - return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, internalNetworksBlock); + return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, networksBlock); } BytesRefVector destinationIpVector = destinationIpBlock.asVector(); if (destinationIpVector == null) { - return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, internalNetworksBlock); + return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, networksBlock); } - return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, internalNetworksBlock).asBlock(); + return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, networksBlock).asBlock(); } } } @@ -76,12 +76,12 @@ public long baseRamBytesUsed() { long baseRamBytesUsed = BASE_RAM_BYTES_USED; baseRamBytesUsed += sourceIp.baseRamBytesUsed(); baseRamBytesUsed += destinationIp.baseRamBytesUsed(); - baseRamBytesUsed += internalNetworks.baseRamBytesUsed(); + baseRamBytesUsed += networks.baseRamBytesUsed(); return baseRamBytesUsed; } public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, - BytesRefBlock destinationIpBlock, BytesRefBlock internalNetworksBlock) { + BytesRefBlock destinationIpBlock, BytesRefBlock networksBlock) { try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { BytesRef sourceIpScratch = new BytesRef(); BytesRef destinationIpScratch = new BytesRef(); @@ -108,12 +108,12 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, result.appendNull(); continue position; } - if (internalNetworksBlock.isNull(p)) { + if (networksBlock.isNull(p)) { result.appendNull(); continue position; } - if (internalNetworksBlock.getValueCount(p) != 1) { - if (internalNetworksBlock.getValueCount(p) > 1) { + if (networksBlock.getValueCount(p) != 1) { + if (networksBlock.getValueCount(p) > 1) { warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); } result.appendNull(); @@ -121,21 +121,21 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, } BytesRef sourceIp = sourceIpBlock.getBytesRef(sourceIpBlock.getFirstValueIndex(p), sourceIpScratch); BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); - result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); + result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, networksBlock)); } return result.build(); } } public BytesRefVector eval(int positionCount, BytesRefVector sourceIpVector, - BytesRefVector destinationIpVector, BytesRefBlock internalNetworksBlock) { + BytesRefVector destinationIpVector, BytesRefBlock networksBlock) { try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { BytesRef sourceIpScratch = new BytesRef(); BytesRef destinationIpScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { BytesRef sourceIp = sourceIpVector.getBytesRef(p, sourceIpScratch); BytesRef destinationIp = destinationIpVector.getBytesRef(p, destinationIpScratch); - result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, internalNetworksBlock)); + result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, networksBlock)); } return result.build(); } @@ -143,12 +143,12 @@ public BytesRefVector eval(int positionCount, BytesRefVector sourceIpVector, @Override public String toString() { - return "NetworkDirectionEvaluator[" + "sourceIp=" + sourceIp + ", destinationIp=" + destinationIp + ", internalNetworks=" + internalNetworks + "]"; + return "NetworkDirectionEvaluator[" + "sourceIp=" + sourceIp + ", destinationIp=" + destinationIp + ", networks=" + networks + "]"; } @Override public void close() { - Releasables.closeExpectNoException(sourceIp, destinationIp, internalNetworks); + Releasables.closeExpectNoException(sourceIp, destinationIp, networks); } private Warnings warnings() { @@ -172,27 +172,27 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory destinationIp; - private final EvalOperator.ExpressionEvaluator.Factory internalNetworks; + private final EvalOperator.ExpressionEvaluator.Factory networks; public Factory(Source source, Function scratch, EvalOperator.ExpressionEvaluator.Factory sourceIp, EvalOperator.ExpressionEvaluator.Factory destinationIp, - EvalOperator.ExpressionEvaluator.Factory internalNetworks) { + EvalOperator.ExpressionEvaluator.Factory networks) { this.source = source; this.scratch = scratch; this.sourceIp = sourceIp; this.destinationIp = destinationIp; - this.internalNetworks = internalNetworks; + this.networks = networks; } @Override public NetworkDirectionEvaluator get(DriverContext context) { - return new NetworkDirectionEvaluator(source, scratch.apply(context), sourceIp.get(context), destinationIp.get(context), internalNetworks.get(context), context); + return new NetworkDirectionEvaluator(source, scratch.apply(context), sourceIp.get(context), destinationIp.get(context), networks.get(context), context); } @Override public String toString() { - return "NetworkDirectionEvaluator[" + "sourceIp=" + sourceIp + ", destinationIp=" + destinationIp + ", internalNetworks=" + internalNetworks + "]"; + return "NetworkDirectionEvaluator[" + "sourceIp=" + sourceIp + ", destinationIp=" + destinationIp + ", networks=" + networks + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index e80d00db04e40..7996fe65a598c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.ip; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -26,6 +27,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -118,7 +120,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua var internalNetworksEvaluatorSupplier = toEvaluator.apply(internalNetworks); return new NetworkDirectionEvaluator.Factory( source(), - context -> new BytesRef(), + context -> new BytesRef(16), sourceIpEvaluatorSupplier, destinationIpEvaluatorSupplier, internalNetworksEvaluatorSupplier @@ -127,9 +129,11 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua @Evaluator static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock networks) { - // Pulling the bytes out directly using InetAddress.getByAddress() requires error handling TODO - InetAddress sourceIpAddress = InetAddresses.forString(sourceIp.utf8ToString()); - InetAddress destinationIpAddress = InetAddresses.forString(destinationIp.utf8ToString()); + System.arraycopy(sourceIp.bytes, sourceIp.offset, scratch.bytes, 0, sourceIp.length); + InetAddress sourceIpAddress = InetAddressPoint.decode(scratch.bytes); + System.arraycopy(destinationIp.bytes, destinationIp.offset, scratch.bytes, 0, destinationIp.length); + InetAddress destinationIpAddress = InetAddressPoint.decode(scratch.bytes); + boolean sourceInternal = false; boolean destinationInternal = false; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 8d69746e6ea44..6c6c24f015b6f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -24,11 +24,34 @@ import java.util.List; import java.util.function.Supplier; +import static org.hamcrest.core.IsEqual.equalTo; + public class NetworkDirectionTests extends AbstractScalarFunctionTestCase { public NetworkDirectionTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } + @ParametersFactory + public static Iterable parameters() { + var suppliers = List.of( + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("10.0.0.0/8"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) + ) + ) + ); + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); + } + @Override protected Expression build(Source source, List args) { return new NetworkDirection(source, args.get(0), args.get(1), args.get(2)); From 1fd1e514bbe9404ff79db01da886e056417689af Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Mon, 6 Oct 2025 13:33:50 -0400 Subject: [PATCH 06/31] Copied over NetworkDirectionUtils tests to the NetworkDirection function tests --- .../NetworkDirectionProcessorTests.java | 1 + .../scalar/ip/NetworkDirectionTests.java | 149 +++++++++++++++++- 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java index af96c287b49a2..a8c5ae0ea2867 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -93,6 +93,7 @@ public void testInvalidNetwork() throws Exception { assertThat(e.getMessage(), containsString("'invalid' is not an IP string literal.")); } + // These tests copy the data from the NetworkDirectionUtils tests public void testCIDR() throws Exception { testNetworkDirectionProcessor(buildEvent("10.0.1.1", "192.168.1.2"), new String[] { "10.0.0.0/8" }, "outbound"); testNetworkDirectionProcessor(buildEvent("192.168.1.2", "10.0.1.1"), new String[] { "10.0.0.0/8" }, "inbound"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 6c6c24f015b6f..ede742252dbd3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.NetworkDirectionUtils; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -33,7 +32,9 @@ public NetworkDirectionTests(@Name("TestCase") Supplier parameters() { + // These tests copy the data from the NetworkDirectionUtils tests var suppliers = List.of( + // CIDR tests new TestCaseSupplier( List.of(DataType.IP, DataType.IP, DataType.KEYWORD), () -> new TestCaseSupplier.TestCase( @@ -46,6 +47,152 @@ public static Iterable parameters() { DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("10.0.0.0/8"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_OUTBOUND)) + ) + ), + // Unspecified tests + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("unspecified"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("unspecified"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + // Private network tests + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + // Public tests + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + List.of(DataType.IP, DataType.IP, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) ) ); From 08647b2f2cce340810da7eb341c79a2d93745b34 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Mon, 6 Oct 2025 13:51:44 -0400 Subject: [PATCH 07/31] Add remaining test types --- .../function/scalar/ip/NetworkDirection.java | 13 +++++- .../scalar/ip/NetworkDirectionErrorTests.java | 41 +++++++++++++++++++ .../NetworkDirectionSerializationTests.java | 41 +++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionSerializationTests.java diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index 7996fe65a598c..ce7a6bdc78c0f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkDirectionUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -160,4 +159,16 @@ static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) Byte public DataType dataType() { return DataType.KEYWORD; } + + public Expression sourceIpField() { + return sourceIpField; + } + + public Expression destinationIpField() { + return destinationIpField; + } + + public Expression internalNetworks() { + return internalNetworks; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java new file mode 100644 index 0000000000000..b15867f765ca1 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class NetworkDirectionErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(NetworkDirectionTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new NetworkDirection(source, args.get(0), args.get(1), args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0, 1 -> "ip"; + case 2 -> "keyword"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionSerializationTests.java new file mode 100644 index 0000000000000..0ef46e8f6d4ca --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionSerializationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class NetworkDirectionSerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected NetworkDirection createTestInstance() { + Source source = randomSource(); + Expression sourceIpField = randomChild(); + Expression destinationIpField = randomChild(); + Expression internalNetworks = randomChild(); + return new NetworkDirection(source, sourceIpField, destinationIpField, internalNetworks); + } + + @Override + protected NetworkDirection mutateInstance(NetworkDirection instance) throws IOException { + Source source = instance.source(); + Expression sourceIpField = instance.sourceIpField(); + Expression destinationIpField = instance.destinationIpField(); + Expression internalNetworks = instance.internalNetworks(); + switch (between(0, 2)) { + case 0 -> sourceIpField = randomValueOtherThan(sourceIpField, AbstractExpressionSerializationTests::randomChild); + case 1 -> destinationIpField = randomValueOtherThan(destinationIpField, AbstractExpressionSerializationTests::randomChild); + case 2 -> internalNetworks = randomValueOtherThan(internalNetworks, AbstractExpressionSerializationTests::randomChild); + } + + return new NetworkDirection(source, sourceIpField, destinationIpField, internalNetworks); + } +} From c4196a272664fcfde985366635ea01071e7020c5 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Mon, 6 Oct 2025 17:13:52 -0400 Subject: [PATCH 08/31] Address the buffer re-use bug; get tests to pass --- .../scalar/ip/NetworkDirectionEvaluator.java | 32 +- .../function/scalar/ip/NetworkDirection.java | 25 +- .../scalar/ip/NetworkDirectionTests.java | 341 +++++++++--------- 3 files changed, 222 insertions(+), 176 deletions(-) diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index d5be28bf7e017..2500229895aae 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -5,6 +5,7 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.ip; import java.lang.IllegalArgumentException; +import java.lang.IllegalStateException; import java.lang.Override; import java.lang.String; import java.util.function.Function; @@ -31,6 +32,8 @@ public final class NetworkDirectionEvaluator implements EvalOperator.ExpressionE private final BytesRef scratch; + private final BytesRef netScratch; + private final EvalOperator.ExpressionEvaluator sourceIp; private final EvalOperator.ExpressionEvaluator destinationIp; @@ -41,11 +44,12 @@ public final class NetworkDirectionEvaluator implements EvalOperator.ExpressionE private Warnings warnings; - public NetworkDirectionEvaluator(Source source, BytesRef scratch, + public NetworkDirectionEvaluator(Source source, BytesRef scratch, BytesRef netScratch, EvalOperator.ExpressionEvaluator sourceIp, EvalOperator.ExpressionEvaluator destinationIp, EvalOperator.ExpressionEvaluator networks, DriverContext driverContext) { this.source = source; this.scratch = scratch; + this.netScratch = netScratch; this.sourceIp = sourceIp; this.destinationIp = destinationIp; this.networks = networks; @@ -65,7 +69,7 @@ public Block eval(Page page) { if (destinationIpVector == null) { return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, networksBlock); } - return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, networksBlock).asBlock(); + return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, networksBlock); } } } @@ -121,21 +125,31 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, } BytesRef sourceIp = sourceIpBlock.getBytesRef(sourceIpBlock.getFirstValueIndex(p), sourceIpScratch); BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); - result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, networksBlock)); + try { + result.appendBytesRef(NetworkDirection.process(this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock)); + } catch (IllegalStateException e) { + warnings().registerException(e); + result.appendNull(); + } } return result.build(); } } - public BytesRefVector eval(int positionCount, BytesRefVector sourceIpVector, + public BytesRefBlock eval(int positionCount, BytesRefVector sourceIpVector, BytesRefVector destinationIpVector, BytesRefBlock networksBlock) { - try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { BytesRef sourceIpScratch = new BytesRef(); BytesRef destinationIpScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { BytesRef sourceIp = sourceIpVector.getBytesRef(p, sourceIpScratch); BytesRef destinationIp = destinationIpVector.getBytesRef(p, destinationIpScratch); - result.appendBytesRef(NetworkDirection.process(this.scratch, sourceIp, destinationIp, p, networksBlock)); + try { + result.appendBytesRef(NetworkDirection.process(this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock)); + } catch (IllegalStateException e) { + warnings().registerException(e); + result.appendNull(); + } } return result.build(); } @@ -168,6 +182,8 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Function scratch; + private final Function netScratch; + private final EvalOperator.ExpressionEvaluator.Factory sourceIp; private final EvalOperator.ExpressionEvaluator.Factory destinationIp; @@ -175,11 +191,13 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory networks; public Factory(Source source, Function scratch, + Function netScratch, EvalOperator.ExpressionEvaluator.Factory sourceIp, EvalOperator.ExpressionEvaluator.Factory destinationIp, EvalOperator.ExpressionEvaluator.Factory networks) { this.source = source; this.scratch = scratch; + this.netScratch = netScratch; this.sourceIp = sourceIp; this.destinationIp = destinationIp; this.networks = networks; @@ -187,7 +205,7 @@ public Factory(Source source, Function scratch, @Override public NetworkDirectionEvaluator get(DriverContext context) { - return new NetworkDirectionEvaluator(source, scratch.apply(context), sourceIp.get(context), destinationIp.get(context), networks.get(context), context); + return new NetworkDirectionEvaluator(source, scratch.apply(context), netScratch.apply(context), sourceIp.get(context), destinationIp.get(context), networks.get(context), context); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index ce7a6bdc78c0f..a29571ad320da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -120,14 +119,22 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua return new NetworkDirectionEvaluator.Factory( source(), context -> new BytesRef(16), + context -> new BytesRef(), sourceIpEvaluatorSupplier, destinationIpEvaluatorSupplier, internalNetworksEvaluatorSupplier ); } - @Evaluator - static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock networks) { + @Evaluator(warnExceptions = IllegalArgumentException.class) + static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef netScratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock networks) { + int valueCount = networks.getValueCount(position); + if (valueCount == 0) { + throw new IllegalArgumentException("List of internal networks must not be empty"); + } + int first = networks.getFirstValueIndex(position); + + System.arraycopy(sourceIp.bytes, sourceIp.offset, scratch.bytes, 0, sourceIp.length); InetAddress sourceIpAddress = InetAddressPoint.decode(scratch.bytes); System.arraycopy(destinationIp.bytes, destinationIp.offset, scratch.bytes, 0, destinationIp.length); @@ -136,17 +143,16 @@ static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) Byte boolean sourceInternal = false; boolean destinationInternal = false; - int valueCount = networks.getValueCount(position); - int first = networks.getFirstValueIndex(position); + // TODO: address scratch re-use due to .length issues above when sharing buffer for (int i = first; i < first + valueCount; i++) { - if (NetworkDirectionUtils.inNetwork(sourceIpAddress, networks.getBytesRef(i, scratch).utf8ToString())) { + if (NetworkDirectionUtils.inNetwork(sourceIpAddress, networks.getBytesRef(i, netScratch).utf8ToString())) { sourceInternal = true; break; } } for (int i = first; i < first + valueCount; i++) { - if (NetworkDirectionUtils.inNetwork(destinationIpAddress, networks.getBytesRef(i, scratch).utf8ToString())) { + if (NetworkDirectionUtils.inNetwork(destinationIpAddress, networks.getBytesRef(i, netScratch).utf8ToString())) { destinationInternal = true; break; } @@ -171,4 +177,9 @@ public Expression destinationIpField() { public Expression internalNetworks() { return internalNetworks; } + + @Override + public boolean foldable() { + return sourceIpField.foldable() && destinationIpField.foldable() && internalNetworks.foldable(); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index ede742252dbd3..91985defb0c44 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -33,170 +34,186 @@ public NetworkDirectionTests(@Name("TestCase") Supplier parameters() { // These tests copy the data from the NetworkDirectionUtils tests - var suppliers = List.of( - // CIDR tests - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("10.0.0.0/8"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("10.0.0.0/8"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_OUTBOUND)) - ) - ), - // Unspecified tests - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("unspecified"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("unspecified"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - // Private network tests - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("private"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - // Public tests - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - List.of(DataType.IP, DataType.IP, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(List.of("public"), DataType.KEYWORD, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + var suppliers = new ArrayList(); + + for (var stringType : DataType.stringTypes()) { + suppliers.addAll(List.of( + // CIDR tests + new TestCaseSupplier( + "CIDR1", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_OUTBOUND)) + ) + ), + new TestCaseSupplier( + "CIDR2", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) + ) + ), + // Unspecified tests + new TestCaseSupplier( + "Unspecified1", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") // TODO: unspecified causes errors + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified2", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + // Private network tests + new TestCaseSupplier( + "Unspecified3", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified4", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified5", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified6", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + // Public tests + new TestCaseSupplier( + "Public1", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + "Public2", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + "Public3", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + "Public4", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) ) - ) - ); + )); + } - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); + return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); } @Override From 295931ab81e48967d6d070d45ca96d6b8027950e Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 7 Oct 2025 14:05:52 -0400 Subject: [PATCH 09/31] Remove to-dos following code review --- .../function/scalar/ip/NetworkDirectionEvaluator.java | 5 ++--- .../esql/expression/function/scalar/ip/NetworkDirection.java | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index 2500229895aae..6f64a680af860 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -5,7 +5,6 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.ip; import java.lang.IllegalArgumentException; -import java.lang.IllegalStateException; import java.lang.Override; import java.lang.String; import java.util.function.Function; @@ -127,7 +126,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); try { result.appendBytesRef(NetworkDirection.process(this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock)); - } catch (IllegalStateException e) { + } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); } @@ -146,7 +145,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector sourceIpVector, BytesRef destinationIp = destinationIpVector.getBytesRef(p, destinationIpScratch); try { result.appendBytesRef(NetworkDirection.process(this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock)); - } catch (IllegalStateException e) { + } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index a29571ad320da..5fc0a41a836fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -143,8 +143,6 @@ static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) Byte boolean sourceInternal = false; boolean destinationInternal = false; - - // TODO: address scratch re-use due to .length issues above when sharing buffer for (int i = first; i < first + valueCount; i++) { if (NetworkDirectionUtils.inNetwork(sourceIpAddress, networks.getBytesRef(i, netScratch).utf8ToString())) { sourceInternal = true; From 584de1732f91b8dcb82961f5dfd8be83d8167ffb Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 7 Oct 2025 14:31:38 -0400 Subject: [PATCH 10/31] Extend csv tests and register as an example --- .../src/main/resources/ip.csv-spec | 19 +++++++++++++++++++ .../function/scalar/ip/NetworkDirection.java | 2 +- .../scalar/ip/NetworkDirectionTests.java | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index afb5baefab7b8..f5415f8af49ef 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -181,6 +181,25 @@ eth0 |gamma |fe80::cae2:65ff:fece:feb9 eth0 |epsilon |[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:fec0, fe80::cae2:65ff:fece:fec1]|fe80::cae2:65ff:fece:fec1|fe80::cae2:65ff:fece:fec1 ; +networkDirectionSimple + +FROM hosts | +WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | +EVAL direction = network_direction(ip0, ip1, ["loopback"]) | +KEEP ip0, ip1, direction +; + + +ip0:ip | ip1:ip | direction:keyword +127.0.0.1 | 127.0.0.1 | internal +::1 | ::1 | internal +127.0.0.1 | ::1 | internal +127.0.0.1 | 127.0.0.2 | internal +127.0.0.1 | 128.0.0.1 | outbound +fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external +fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound +; + cidrMatchSimple from hosts | where cidr_match(ip1, "127.0.0.2/32") | keep card, host, ip0, ip1; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index 5fc0a41a836fa..ca2638820fe01 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -52,7 +52,7 @@ public class NetworkDirection extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", description = "Returns true if the direction of the source-to-destination-IPs is determined to be inbound, provided a list of internal networks.", - examples = @Example(file = "ip", tag = "cdirMatchMultipleArgs") // TODO make an example + examples = @Example(file = "ip", tag = "networkDirectionSimple") ) public NetworkDirection( Source source, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 91985defb0c44..4df324198fc95 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -75,7 +75,7 @@ public static Iterable parameters() { List.of( new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "source_ip"), new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") // TODO: unspecified causes errors + new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") ), "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", DataType.KEYWORD, From 1502b71af82e00815bb5f5a4c2767267b00b79d7 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 7 Oct 2025 15:18:43 -0400 Subject: [PATCH 11/31] Docs and CSV test, singular --- .../description/network_direction.md | 6 ++ .../functions/examples/network_direction.md | 22 +++++++ .../functions/layout/network_direction.md | 23 +++++++ .../functions/parameters/network_direction.md | 13 ++++ .../functions/types/network_direction.md | 9 +++ .../images/functions/network_direction.svg | 1 + .../functions/network_direction.json | 61 +++++++++++++++++++ .../docs/functions/network_direction.md | 11 ++++ .../src/main/resources/ip.csv-spec | 42 +++++++------ .../function/scalar/ip/NetworkDirection.java | 4 +- 10 files changed, 171 insertions(+), 21 deletions(-) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/network_direction.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/network_direction.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/parameters/network_direction.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/network_direction.md create mode 100644 docs/reference/query-languages/esql/images/functions/network_direction.svg create mode 100644 docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json create mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/description/network_direction.md new file mode 100644 index 0000000000000..8eb58c0735b86 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/network_direction.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md new file mode 100644 index 0000000000000..f80f1011a9cc4 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md @@ -0,0 +1,22 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql +FROM hosts | +WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | +EVAL direction = network_direction(ip0, ip1, ["loopback"]) | +KEEP ip0, ip1, direction +``` + +| ip0:ip | ip1:ip | direction:keyword | +| --- | --- | --- | +| 127.0.0.1 | 127.0.0.1 | internal | +| ::1 | ::1 | internal | +| 127.0.0.1 | ::1 | internal | +| 127.0.0.1 | 127.0.0.2 | internal | +| 127.0.0.1 | 128.0.0.1 | outbound | +| fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external | +| fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/layout/network_direction.md new file mode 100644 index 0000000000000..c7d72dfece8d4 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/network_direction.md @@ -0,0 +1,23 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `NETWORK_DIRECTION` [esql-network_direction] + +**Syntax** + +:::{image} ../../../images/functions/network_direction.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/network_direction.md +::: + +:::{include} ../description/network_direction.md +::: + +:::{include} ../types/network_direction.md +::: + +:::{include} ../examples/network_direction.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/network_direction.md new file mode 100644 index 0000000000000..ce78ff7b8b4a0 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/network_direction.md @@ -0,0 +1,13 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`source_ip` +: Source IP address of type `ip` (both IPv4 and IPv6 are supported). + +`destination_ip` +: Destination IP address of type `ip` (both IPv4 and IPv6 are supported). + +`internal_networks` +: List of internal networks. Supports IPv4 and IPv6 addresses, ranges in CIDR notation, and named ranges. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/types/network_direction.md new file mode 100644 index 0000000000000..ea29dc5d04589 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/network_direction.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| source_ip | destination_ip | internal_networks | result | +| --- | --- | --- | --- | +| ip | ip | keyword | keyword | +| ip | ip | text | keyword | + diff --git a/docs/reference/query-languages/esql/images/functions/network_direction.svg b/docs/reference/query-languages/esql/images/functions/network_direction.svg new file mode 100644 index 0000000000000..9f65a55a81523 --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/network_direction.svg @@ -0,0 +1 @@ +NETWORK_DIRECTION(source_ip,destination_ip,internal_networks) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json new file mode 100644 index 0000000000000..f6fe7369ea5a6 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "scalar", + "name" : "network_direction", + "description" : "Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks.", + "signatures" : [ + { + "params" : [ + { + "name" : "source_ip", + "type" : "ip", + "optional" : false, + "description" : "Source IP address of type `ip` (both IPv4 and IPv6 are supported)." + }, + { + "name" : "destination_ip", + "type" : "ip", + "optional" : false, + "description" : "Destination IP address of type `ip` (both IPv4 and IPv6 are supported)." + }, + { + "name" : "internal_networks", + "type" : "keyword", + "optional" : false, + "description" : "List of internal networks. Supports IPv4 and IPv6 addresses, ranges in CIDR notation, and named ranges." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "source_ip", + "type" : "ip", + "optional" : false, + "description" : "Source IP address of type `ip` (both IPv4 and IPv6 are supported)." + }, + { + "name" : "destination_ip", + "type" : "ip", + "optional" : false, + "description" : "Destination IP address of type `ip` (both IPv4 and IPv6 are supported)." + }, + { + "name" : "internal_networks", + "type" : "text", + "optional" : false, + "description" : "List of internal networks. Supports IPv4 and IPv6 addresses, ranges in CIDR notation, and named ranges." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM hosts |\nWHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 |\nEVAL direction = network_direction(ip0, ip1, [\"loopback\"]) |\nKEEP ip0, ip1, direction" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md new file mode 100644 index 0000000000000..0598b518985b5 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md @@ -0,0 +1,11 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### NETWORK DIRECTION +Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. + +```esql +FROM hosts | +WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | +EVAL direction = network_direction(ip0, ip1, ["loopback"]) | +KEEP ip0, ip1, direction +``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index f5415f8af49ef..1c4035bf340f4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -181,25 +181,6 @@ eth0 |gamma |fe80::cae2:65ff:fece:feb9 eth0 |epsilon |[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:fec0, fe80::cae2:65ff:fece:fec1]|fe80::cae2:65ff:fece:fec1|fe80::cae2:65ff:fece:fec1 ; -networkDirectionSimple - -FROM hosts | -WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | -EVAL direction = network_direction(ip0, ip1, ["loopback"]) | -KEEP ip0, ip1, direction -; - - -ip0:ip | ip1:ip | direction:keyword -127.0.0.1 | 127.0.0.1 | internal -::1 | ::1 | internal -127.0.0.1 | ::1 | internal -127.0.0.1 | 127.0.0.2 | internal -127.0.0.1 | 128.0.0.1 | outbound -fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external -fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound -; - cidrMatchSimple from hosts | where cidr_match(ip1, "127.0.0.2/32") | keep card, host, ip0, ip1; @@ -748,3 +729,26 @@ row ip4 = to_ip("1.2.3.4") ip4:ip | a:ip | b:ip | c:ip 1.2.3.4 | null | null | null ; + +networkDirectionSimple +required_capability: network_direction + +// tag::networkDirectionSimple[] +FROM hosts | +WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | +EVAL direction = network_direction(ip0, ip1, ["loopback"]) | +KEEP ip0, ip1, direction +// end::networkDirectionSimple[] +; + +// tag::networkDirectionSimple-result[] +ip0:ip | ip1:ip | direction:keyword +127.0.0.1 | 127.0.0.1 | internal +::1 | ::1 | internal +127.0.0.1 | ::1 | internal +127.0.0.1 | 127.0.0.2 | internal +127.0.0.1 | 128.0.0.1 | outbound +fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external +fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound +// end::networkDirectionSimple-result[] +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index ca2638820fe01..5b981c3ab6e74 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -36,7 +36,7 @@ import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL; /** - * Returns whether a connection is inbound given a source IP address, destination IP address, and a list of internal networks. + * Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. */ public class NetworkDirection extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -51,7 +51,7 @@ public class NetworkDirection extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", - description = "Returns true if the direction of the source-to-destination-IPs is determined to be inbound, provided a list of internal networks.", + description = "Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks.", examples = @Example(file = "ip", tag = "networkDirectionSimple") ) public NetworkDirection( From 737eae0f895de2def72a0426bba8d5891d80dd17 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 7 Oct 2025 16:00:57 -0400 Subject: [PATCH 12/31] Tests work great now --- .../scalar/ip/NetworkDirectionEvaluator.java | 48 +++---------------- .../function/scalar/ip/NetworkDirection.java | 27 +++++++++-- .../scalar/ip/NetworkDirectionErrorTests.java | 2 +- .../scalar/ip/NetworkDirectionTests.java | 4 +- 4 files changed, 32 insertions(+), 49 deletions(-) diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index 6f64a680af860..5dd8d477ba2b9 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -12,7 +12,6 @@ import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; @@ -60,15 +59,7 @@ public Block eval(Page page) { try (BytesRefBlock sourceIpBlock = (BytesRefBlock) sourceIp.eval(page)) { try (BytesRefBlock destinationIpBlock = (BytesRefBlock) destinationIp.eval(page)) { try (BytesRefBlock networksBlock = (BytesRefBlock) networks.eval(page)) { - BytesRefVector sourceIpVector = sourceIpBlock.asVector(); - if (sourceIpVector == null) { - return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, networksBlock); - } - BytesRefVector destinationIpVector = destinationIpBlock.asVector(); - if (destinationIpVector == null) { - return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, networksBlock); - } - return eval(page.getPositionCount(), sourceIpVector, destinationIpVector, networksBlock); + return eval(page.getPositionCount(), sourceIpBlock, destinationIpBlock, networksBlock); } } } @@ -89,6 +80,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, BytesRef sourceIpScratch = new BytesRef(); BytesRef destinationIpScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; if (sourceIpBlock.isNull(p)) { result.appendNull(); continue position; @@ -111,44 +103,16 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, result.appendNull(); continue position; } - if (networksBlock.isNull(p)) { - result.appendNull(); - continue position; + if (!networksBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (networksBlock.getValueCount(p) != 1) { - if (networksBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } + if (allBlocksAreNulls) { result.appendNull(); continue position; } BytesRef sourceIp = sourceIpBlock.getBytesRef(sourceIpBlock.getFirstValueIndex(p), sourceIpScratch); BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); - try { - result.appendBytesRef(NetworkDirection.process(this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock)); - } catch (IllegalArgumentException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public BytesRefBlock eval(int positionCount, BytesRefVector sourceIpVector, - BytesRefVector destinationIpVector, BytesRefBlock networksBlock) { - try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { - BytesRef sourceIpScratch = new BytesRef(); - BytesRef destinationIpScratch = new BytesRef(); - position: for (int p = 0; p < positionCount; p++) { - BytesRef sourceIp = sourceIpVector.getBytesRef(p, sourceIpScratch); - BytesRef destinationIp = destinationIpVector.getBytesRef(p, destinationIpScratch); - try { - result.appendBytesRef(NetworkDirection.process(this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock)); - } catch (IllegalArgumentException e) { - warnings().registerException(e); - result.appendNull(); - } + NetworkDirection.process(result, this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock); } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index 5b981c3ab6e74..ad0a95722485d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -34,6 +34,12 @@ import java.util.List; import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.fromIndex; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isIPAndExact; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; /** * Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. @@ -126,15 +132,15 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua ); } - @Evaluator(warnExceptions = IllegalArgumentException.class) - static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef netScratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock networks) { + @Evaluator() + static void process(BytesRefBlock.Builder builder, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef netScratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock networks) { int valueCount = networks.getValueCount(position); if (valueCount == 0) { - throw new IllegalArgumentException("List of internal networks must not be empty"); + builder.appendNull(); + return; } int first = networks.getFirstValueIndex(position); - System.arraycopy(sourceIp.bytes, sourceIp.offset, scratch.bytes, 0, sourceIp.length); InetAddress sourceIpAddress = InetAddressPoint.decode(scratch.bytes); System.arraycopy(destinationIp.bytes, destinationIp.offset, scratch.bytes, 0, destinationIp.length); @@ -156,7 +162,7 @@ static BytesRef process(@Fixed(includeInToString=false, scope=THREAD_LOCAL) Byte } } - return new BytesRef(NetworkDirectionUtils.getDirection(sourceInternal, destinationInternal)); + builder.appendBytesRef(new BytesRef(NetworkDirectionUtils.getDirection(sourceInternal, destinationInternal))); } @Override @@ -164,6 +170,17 @@ public DataType dataType() { return DataType.KEYWORD; } + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isIPAndExact(sourceIpField, sourceText(), FIRST) + .and(isIPAndExact(destinationIpField, sourceText(), SECOND)) + .and(isStringAndExact(internalNetworks, sourceText(), THIRD)); + } + public Expression sourceIpField() { return sourceIpField; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java index b15867f765ca1..0ba8fff3bdb73 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionErrorTests.java @@ -34,7 +34,7 @@ protected Expression build(Source source, List args) { protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 0, 1 -> "ip"; - case 2 -> "keyword"; + case 2 -> "string"; default -> ""; })); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 4df324198fc95..d314d628df5d1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.function.Supplier; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.IsEqual.equalTo; public class NetworkDirectionTests extends AbstractScalarFunctionTestCase { @@ -34,7 +35,7 @@ public NetworkDirectionTests(@Name("TestCase") Supplier parameters() { // These tests copy the data from the NetworkDirectionUtils tests - var suppliers = new ArrayList(); + List suppliers = new ArrayList<>(); for (var stringType : DataType.stringTypes()) { suppliers.addAll(List.of( @@ -212,6 +213,7 @@ public static Iterable parameters() { ) )); } + suppliers = anyNullIsNull(true, suppliers); return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); } From b80ef4924c74f57ce5f0f09c717ff32ca96303b9 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 7 Oct 2025 16:23:00 -0400 Subject: [PATCH 13/31] Change up some of the CSV tests --- .../functions/examples/network_direction.md | 15 ++------ .../functions/network_direction.json | 2 +- .../docs/functions/network_direction.md | 7 ++-- .../src/main/resources/ip.csv-spec | 37 +++++++++++++------ .../function/scalar/ip/NetworkDirection.java | 2 +- 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md index f80f1011a9cc4..6da1f18d11948 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md @@ -3,20 +3,13 @@ **Example** ```esql -FROM hosts | -WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | -EVAL direction = network_direction(ip0, ip1, ["loopback"]) | -KEEP ip0, ip1, direction +ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] +| EVAL direction = network_direction(ip0, ip1, networks) +| DROP networks ``` | ip0:ip | ip1:ip | direction:keyword | | --- | --- | --- | -| 127.0.0.1 | 127.0.0.1 | internal | -| ::1 | ::1 | internal | -| 127.0.0.1 | ::1 | internal | -| 127.0.0.1 | 127.0.0.2 | internal | -| 127.0.0.1 | 128.0.0.1 | outbound | -| fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external | -| fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound | +| 1.2.3.4 | 5.6.7.8 | external | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json index f6fe7369ea5a6..74793831404b0 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json @@ -54,7 +54,7 @@ } ], "examples" : [ - "FROM hosts |\nWHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 |\nEVAL direction = network_direction(ip0, ip1, [\"loopback\"]) |\nKEEP ip0, ip1, direction" + "ROW ip0 = \"1.2.3.4\"::ip, ip1 = \"5.6.7.8\"::ip, networks = [\"loopback\", \"private\"]\n| EVAL direction = network_direction(ip0, ip1, networks)\n| DROP networks" ], "preview" : false, "snapshot_only" : false diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md index 0598b518985b5..0c14bf1ec0541 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md @@ -4,8 +4,7 @@ Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. ```esql -FROM hosts | -WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | -EVAL direction = network_direction(ip0, ip1, ["loopback"]) | -KEEP ip0, ip1, direction +ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] +| EVAL direction = network_direction(ip0, ip1, networks) +| DROP networks ``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index 1c4035bf340f4..e1ece819164ff 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -733,22 +733,35 @@ ip4:ip | a:ip | b:ip | c:ip networkDirectionSimple required_capability: network_direction -// tag::networkDirectionSimple[] -FROM hosts | -WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | -EVAL direction = network_direction(ip0, ip1, ["loopback"]) | -KEEP ip0, ip1, direction -// end::networkDirectionSimple[] +FROM hosts +| WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 +| EVAL direction = network_direction(ip0, ip1, ["loopback"]) +| KEEP ip0, ip1, direction +| SORT direction ; -// tag::networkDirectionSimple-result[] ip0:ip | ip1:ip | direction:keyword -127.0.0.1 | 127.0.0.1 | internal -::1 | ::1 | internal +fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external +fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound 127.0.0.1 | ::1 | internal 127.0.0.1 | 127.0.0.2 | internal +::1 | ::1 | internal +127.0.0.1 | 127.0.0.1 | internal 127.0.0.1 | 128.0.0.1 | outbound -fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external -fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound -// end::networkDirectionSimple-result[] +; + +networkDirectionFromRow +required_capability: network_direction + +// tag::networkDirectionFromRow[] +ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] +| EVAL direction = network_direction(ip0, ip1, networks) +| DROP networks +// end::networkDirectionFromRow[] +; + +// tag::networkDirectionFromRow-result[] +ip0:ip | ip1:ip | direction:keyword +1.2.3.4 | 5.6.7.8 | external +// end::networkDirectionFromRow-result[] ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index ad0a95722485d..31c043c081c45 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -58,7 +58,7 @@ public class NetworkDirection extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", description = "Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks.", - examples = @Example(file = "ip", tag = "networkDirectionSimple") + examples = @Example(file = "ip", tag = "networkDirectionFromRow") ) public NetworkDirection( Source source, From 5e245123d250b1e542ddf146758e6128eb012839 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 7 Oct 2025 16:25:56 -0400 Subject: [PATCH 14/31] Change up some of the CSV tests pt. 2 --- .../functions/examples/network_direction.md | 5 ++--- .../functions/network_direction.json | 2 +- .../docs/functions/network_direction.md | 5 ++--- .../src/main/resources/ip.csv-spec | 19 +++++++++++++++---- .../function/scalar/ip/NetworkDirection.java | 2 +- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md index 6da1f18d11948..925a81bb2040b 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md @@ -3,9 +3,8 @@ **Example** ```esql -ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] -| EVAL direction = network_direction(ip0, ip1, networks) -| DROP networks +ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip +| EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) ``` | ip0:ip | ip1:ip | direction:keyword | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json index 74793831404b0..124a3173c81e3 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json @@ -54,7 +54,7 @@ } ], "examples" : [ - "ROW ip0 = \"1.2.3.4\"::ip, ip1 = \"5.6.7.8\"::ip, networks = [\"loopback\", \"private\"]\n| EVAL direction = network_direction(ip0, ip1, networks)\n| DROP networks" + "ROW ip0 = \"1.2.3.4\"::ip, ip1 = \"5.6.7.8\"::ip\n| EVAL direction = network_direction(ip0, ip1, [\"loopback\", \"private\"])" ], "preview" : false, "snapshot_only" : false diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md index 0c14bf1ec0541..62cc0aeb82240 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md @@ -4,7 +4,6 @@ Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. ```esql -ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] -| EVAL direction = network_direction(ip0, ip1, networks) -| DROP networks +ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip +| EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) ``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index e1ece819164ff..c6644ebe1674b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -753,15 +753,26 @@ fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound networkDirectionFromRow required_capability: network_direction -// tag::networkDirectionFromRow[] ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] | EVAL direction = network_direction(ip0, ip1, networks) | DROP networks -// end::networkDirectionFromRow[] ; -// tag::networkDirectionFromRow-result[] ip0:ip | ip1:ip | direction:keyword 1.2.3.4 | 5.6.7.8 | external -// end::networkDirectionFromRow-result[] +; + +networkDirectionFromRowWithInlineNetworks +required_capability: network_direction + +// tag::networkDirectionFromRowWithInlineNetworks[] +ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip +| EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) +// end::networkDirectionFromRowWithInlineNetworks[] +; + +// tag::networkDirectionFromRowWithInlineNetworks-result[] +ip0:ip | ip1:ip | direction:keyword +1.2.3.4 | 5.6.7.8 | external +// end::networkDirectionFromRowWithInlineNetworks-result[] ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index 31c043c081c45..d4b13f7ffb580 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -58,7 +58,7 @@ public class NetworkDirection extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", description = "Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks.", - examples = @Example(file = "ip", tag = "networkDirectionFromRow") + examples = @Example(file = "ip", tag = "networkDirectionFromRowWithInlineNetworks") ) public NetworkDirection( Source source, From f98b9e056180309368a477fc3b7778bdb61f4e12 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 7 Oct 2025 16:29:11 -0400 Subject: [PATCH 15/31] Spotless gets it done --- .../network/NetworkDirectionUtilsTests.java | 8 +- .../function/scalar/ip/NetworkDirection.java | 19 +- .../scalar/ip/NetworkDirectionTests.java | 384 ++++++++++-------- 3 files changed, 226 insertions(+), 185 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java b/server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java index 4b2c9f1381326..38bd908920897 100644 --- a/server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/common/network/NetworkDirectionUtilsTests.java @@ -16,24 +16,24 @@ import static org.hamcrest.Matchers.equalTo; public class NetworkDirectionUtilsTests extends ESTestCase { - public void testCIDR() { + public void testCIDR() { testNetworkDirectionUtils("10.0.1.1", "192.168.1.2", List.of("10.0.0.0/8"), "outbound"); testNetworkDirectionUtils("192.168.1.2", "10.0.1.1", List.of("10.0.0.0/8"), "inbound"); } - public void testUnspecified() { + public void testUnspecified() { testNetworkDirectionUtils("0.0.0.0", "0.0.0.0", List.of("unspecified"), "internal"); testNetworkDirectionUtils("::", "::", List.of("unspecified"), "internal"); } - public void testNetworkPrivate() { + public void testNetworkPrivate() { testNetworkDirectionUtils("192.168.1.1", "192.168.1.2", List.of("private"), "internal"); testNetworkDirectionUtils("10.0.1.1", "192.168.1.2", List.of("private"), "internal"); testNetworkDirectionUtils("192.168.1.1", "172.16.0.1", List.of("private"), "internal"); testNetworkDirectionUtils("192.168.1.1", "fd12:3456:789a:1::1", List.of("private"), "internal"); } - public void testNetworkPublic() { + public void testNetworkPublic() { testNetworkDirectionUtils("192.168.1.1", "192.168.1.2", List.of("public"), "external"); testNetworkDirectionUtils("10.0.1.1", "192.168.1.2", List.of("public"), "external"); testNetworkDirectionUtils("192.168.1.1", "172.16.0.1", List.of("public"), "external"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index d4b13f7ffb580..46c5590ea46fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -37,7 +37,6 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.fromIndex; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isIPAndExact; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; @@ -75,8 +74,8 @@ public NetworkDirection( @Param( name = "internal_networks", type = { "keyword", "text" }, - description = "List of internal networks. Supports IPv4 and IPv6 addresses, ranges in CIDR notation, and named ranges.") - Expression internalNetworks + description = "List of internal networks. Supports IPv4 and IPv6 addresses, ranges in CIDR notation, and named ranges." + ) Expression internalNetworks ) { super(source, Arrays.asList(sourceIpField, destinationIpField, internalNetworks)); this.sourceIpField = sourceIpField; @@ -116,7 +115,6 @@ protected NodeInfo info() { return NodeInfo.create(this, NetworkDirection::new, sourceIpField, destinationIpField, internalNetworks); } - @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var sourceIpEvaluatorSupplier = toEvaluator.apply(sourceIpField); @@ -133,7 +131,15 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua } @Evaluator() - static void process(BytesRefBlock.Builder builder, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef scratch, @Fixed(includeInToString=false, scope=THREAD_LOCAL) BytesRef netScratch, BytesRef sourceIp, BytesRef destinationIp, @Position int position, BytesRefBlock networks) { + static void process( + BytesRefBlock.Builder builder, + @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef scratch, + @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef netScratch, + BytesRef sourceIp, + BytesRef destinationIp, + @Position int position, + BytesRefBlock networks + ) { int valueCount = networks.getValueCount(position); if (valueCount == 0) { builder.appendNull(); @@ -176,8 +182,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - return isIPAndExact(sourceIpField, sourceText(), FIRST) - .and(isIPAndExact(destinationIpField, sourceText(), SECOND)) + return isIPAndExact(sourceIpField, sourceText(), FIRST).and(isIPAndExact(destinationIpField, sourceText(), SECOND)) .and(isStringAndExact(internalNetworks, sourceText(), THIRD)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index d314d628df5d1..05999477a987c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.ip; import com.carrotsearch.randomizedtesting.annotations.Name; - import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; @@ -24,7 +23,6 @@ import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.IsEqual.equalTo; public class NetworkDirectionTests extends AbstractScalarFunctionTestCase { @@ -38,180 +36,218 @@ public static Iterable parameters() { List suppliers = new ArrayList<>(); for (var stringType : DataType.stringTypes()) { - suppliers.addAll(List.of( - // CIDR tests - new TestCaseSupplier( - "CIDR1", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_OUTBOUND)) - ) - ), - new TestCaseSupplier( - "CIDR2", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) - ) - ), - // Unspecified tests - new TestCaseSupplier( - "Unspecified1", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - "Unspecified2", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - // Private network tests - new TestCaseSupplier( - "Unspecified3", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - "Unspecified4", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - "Unspecified5", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - "Unspecified6", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - // Public tests - new TestCaseSupplier( - "Public1", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - "Public2", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - "Public3", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("172.16.0.1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - "Public4", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + suppliers.addAll( + List.of( + // CIDR tests + new TestCaseSupplier( + "CIDR1", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("192.168.1.2"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_OUTBOUND)) + ) + ), + new TestCaseSupplier( + "CIDR2", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) + ) + ), + // Unspecified tests + new TestCaseSupplier( + "Unspecified1", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified2", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + // Private network tests + new TestCaseSupplier( + "Unspecified3", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("192.168.1.2"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified4", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("192.168.1.2"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified5", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("172.16.0.1"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + new TestCaseSupplier( + "Unspecified6", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) + ) + ), + // Public tests + new TestCaseSupplier( + "Public1", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("192.168.1.2"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + "Public2", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("192.168.1.2"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + "Public3", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("172.16.0.1"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) + ), + new TestCaseSupplier( + "Public4", + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData( + EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), + DataType.IP, + "destination_ip" + ), + new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) + ) ) ) - )); + ); } suppliers = anyNullIsNull(true, suppliers); From 0e7ba5547d2fab3fd980181bc638c86fedb75fc6 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 7 Oct 2025 17:03:09 -0400 Subject: [PATCH 16/31] Update docs/changelog/136133.yaml --- docs/changelog/136133.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/136133.yaml diff --git a/docs/changelog/136133.yaml b/docs/changelog/136133.yaml new file mode 100644 index 0000000000000..564aa615da610 --- /dev/null +++ b/docs/changelog/136133.yaml @@ -0,0 +1,5 @@ +pr: 136133 +summary: Implement `network_direction` function +area: ES|QL +type: enhancement +issues: [] From 463cd790bd09680ac312fe11f4131a2d67a4c3a4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 7 Oct 2025 21:14:07 +0000 Subject: [PATCH 17/31] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 2e2670d42c3bc..89e97d4c1486d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1477,7 +1477,7 @@ public enum Cap { * Support for requesting the "_tsid" metadata field. */ METADATA_TSID_FIELD, - + /** * Fix management of plans with no columns * https://github.com/elastic/elasticsearch/issues/120272 @@ -1488,7 +1488,7 @@ public enum Cap { * Support for dots in FUSE attributes */ DOTS_IN_FUSE, - + /** * Network direction function. */ From d810ecff4d2a420429b1d562378773de86ca4d90 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Wed, 8 Oct 2025 10:28:21 -0400 Subject: [PATCH 18/31] Fix sorting of expected outputs for multicluster CSV IP run --- .../esql/qa/testFixtures/src/main/resources/ip.csv-spec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index c6644ebe1674b..df4252479cc8e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -737,16 +737,16 @@ FROM hosts | WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 | EVAL direction = network_direction(ip0, ip1, ["loopback"]) | KEEP ip0, ip1, direction -| SORT direction +| SORT direction, ip0, ip1 ; ip0:ip | ip1:ip | direction:keyword fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | external fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | inbound -127.0.0.1 | ::1 | internal -127.0.0.1 | 127.0.0.2 | internal ::1 | ::1 | internal +127.0.0.1 | ::1 | internal 127.0.0.1 | 127.0.0.1 | internal +127.0.0.1 | 127.0.0.2 | internal 127.0.0.1 | 128.0.0.1 | outbound ; From af8038caa93cbc55165efc39df79874f8f4adefd Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Wed, 8 Oct 2025 10:39:56 -0400 Subject: [PATCH 19/31] Cleaning up checkstyle violations --- .../expression/function/scalar/ip/NetworkDirection.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index 46c5590ea46fb..dd44f30256eac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -41,7 +41,8 @@ import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; /** - * Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. + * Returns the direction type (inbound, outbound, internal, external) given + * a source IP address, destination IP address, and a list of internal networks. */ public class NetworkDirection extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -56,7 +57,8 @@ public class NetworkDirection extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", - description = "Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks.", + description = "Returns the direction type (inbound, outbound, internal, external) given " + + "a source IP address, destination IP address, and a list of internal networks.", examples = @Example(file = "ip", tag = "networkDirectionFromRowWithInlineNetworks") ) public NetworkDirection( From d788b521e6781607ef458bc0af629291a2c78f0a Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Wed, 8 Oct 2025 10:44:10 -0400 Subject: [PATCH 20/31] Modify function usage example in CSV tests --- .../esql/_snippets/functions/examples/network_direction.md | 4 ++-- .../esql/kibana/definition/functions/network_direction.json | 2 +- .../esql/kibana/docs/functions/network_direction.md | 2 +- .../esql/qa/testFixtures/src/main/resources/ip.csv-spec | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md index 925a81bb2040b..8205925475594 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md @@ -3,12 +3,12 @@ **Example** ```esql -ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip +ROW ip0 = "127.0.0.1"::ip, ip1 = "5.6.7.8"::ip | EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) ``` | ip0:ip | ip1:ip | direction:keyword | | --- | --- | --- | -| 1.2.3.4 | 5.6.7.8 | external | +| 127.0.0.1 | 5.6.7.8 | outbound | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json index 124a3173c81e3..29910e16cd336 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json @@ -54,7 +54,7 @@ } ], "examples" : [ - "ROW ip0 = \"1.2.3.4\"::ip, ip1 = \"5.6.7.8\"::ip\n| EVAL direction = network_direction(ip0, ip1, [\"loopback\", \"private\"])" + "ROW ip0 = \"127.0.0.1\"::ip, ip1 = \"5.6.7.8\"::ip\n| EVAL direction = network_direction(ip0, ip1, [\"loopback\", \"private\"])" ], "preview" : false, "snapshot_only" : false diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md index 62cc0aeb82240..8746c2cc7e033 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md @@ -4,6 +4,6 @@ Returns the direction type (inbound, outbound, internal, external) given a source IP address, destination IP address, and a list of internal networks. ```esql -ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip +ROW ip0 = "127.0.0.1"::ip, ip1 = "5.6.7.8"::ip | EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) ``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index df4252479cc8e..bf1f4e56fb6fd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -766,13 +766,13 @@ networkDirectionFromRowWithInlineNetworks required_capability: network_direction // tag::networkDirectionFromRowWithInlineNetworks[] -ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip +ROW ip0 = "127.0.0.1"::ip, ip1 = "5.6.7.8"::ip | EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) // end::networkDirectionFromRowWithInlineNetworks[] ; // tag::networkDirectionFromRowWithInlineNetworks-result[] -ip0:ip | ip1:ip | direction:keyword -1.2.3.4 | 5.6.7.8 | external +ip0:ip | ip1:ip | direction:keyword +127.0.0.1 | 5.6.7.8 | outbound // end::networkDirectionFromRowWithInlineNetworks-result[] ; From 22d301a67398dc0d26da146f6355f28c8d1ddd3b Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Wed, 8 Oct 2025 11:10:22 -0400 Subject: [PATCH 21/31] Update test formatting as well --- .../scalar/ip/NetworkDirectionTests.java | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 05999477a987c..467ed8543b066 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -52,7 +52,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_OUTBOUND)) ) @@ -66,7 +67,8 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "destination_ip"), new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) ) @@ -81,7 +83,8 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) ) @@ -95,7 +98,8 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "destination_ip"), new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) ) @@ -114,7 +118,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) ) @@ -132,7 +137,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) ) @@ -150,7 +156,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) ) @@ -168,7 +175,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) ) @@ -187,7 +195,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) ) @@ -205,7 +214,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) ) @@ -223,7 +233,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) ) @@ -241,7 +252,8 @@ public static Iterable parameters() { ), new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1], networks=Attribute[channel=2]]", + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", DataType.KEYWORD, equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) ) From 095106cf877ef14f626b2c0fa0b60d704b6e3925 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 14 Oct 2025 10:01:32 -0400 Subject: [PATCH 22/31] Create test case creaion helper method --- .../scalar/ip/NetworkDirectionTests.java | 253 +++--------------- 1 file changed, 40 insertions(+), 213 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 467ed8543b066..951a75303d944 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -39,224 +39,26 @@ public static Iterable parameters() { suppliers.addAll( List.of( // CIDR tests - new TestCaseSupplier( - "CIDR1", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("192.168.1.2"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_OUTBOUND)) - ) - ), - new TestCaseSupplier( - "CIDR2", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.2"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/8"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INBOUND)) - ) - ), + createTestCase("CIDR1", "10.0.1.1", "192.168.1.2", "10.0.0.0/8", stringType, NetworkDirectionUtils.DIRECTION_OUTBOUND), + createTestCase("CIDR2", "192.168.1.2", "10.0.1.1", "10.0.0.0/8", stringType, NetworkDirectionUtils.DIRECTION_INBOUND), // Unspecified tests - new TestCaseSupplier( - "Unspecified1", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("0.0.0.0"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - "Unspecified2", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::"), DataType.IP, "destination_ip"), - new TestCaseSupplier.TypedData(new BytesRef("unspecified"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), + createTestCase("Unspecified1", "0.0.0.0", "0.0.0.0", "unspecified", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), + createTestCase("Unspecified2", "::", "::", "unspecified", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), // Private network tests - new TestCaseSupplier( - "Unspecified3", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("192.168.1.2"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - "Unspecified4", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("192.168.1.2"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) - ), - new TestCaseSupplier( - "Unspecified5", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("172.16.0.1"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) + createTestCase( + "Private1", "192.168.1.1", "192.168.1.2", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL ), - new TestCaseSupplier( - "Unspecified6", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("private"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_INTERNAL)) - ) + createTestCase("Private2", "10.0.1.1", "192.168.1.2", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), + createTestCase("Private3", "192.168.1.1", "172.16.0.1", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), + createTestCase( + "Private4", "192.168.1.1", "fd12:3456:789a:1::1", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL ), // Public tests - new TestCaseSupplier( - "Public1", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("192.168.1.2"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - "Public2", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("10.0.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("192.168.1.2"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - "Public3", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("172.16.0.1"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) - ), - new TestCaseSupplier( - "Public4", - List.of(DataType.IP, DataType.IP, stringType), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.1.1"), DataType.IP, "source_ip"), - new TestCaseSupplier.TypedData( - EsqlDataTypeConverter.stringToIP("fd12:3456:789a:1::1"), - DataType.IP, - "destination_ip" - ), - new TestCaseSupplier.TypedData(new BytesRef("public"), stringType, "internal_networks") - ), - "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," - + " networks=Attribute[channel=2]]", - DataType.KEYWORD, - equalTo(new BytesRef(NetworkDirectionUtils.DIRECTION_EXTERNAL)) - ) + createTestCase("Public1", "192.168.1.1", "192.168.1.2", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL), + createTestCase("Public2", "10.0.1.1", "192.168.1.2", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL), + createTestCase("Public3", "192.168.1.1", "172.16.0.1", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL), + createTestCase( + "Public4", "192.168.1.1", "fd12:3456:789a:1::1", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL ) ) ); @@ -266,6 +68,31 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); } + private static TestCaseSupplier createTestCase( + String testName, + String sourceIp, + String destinationIp, + String internalNetworks, + DataType stringType, + String expectedDirection + ) { + return new TestCaseSupplier( + testName, + List.of(DataType.IP, DataType.IP, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP(sourceIp), DataType.IP, "source_ip"), + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP(destinationIp), DataType.IP, "destination_ip"), + new TestCaseSupplier.TypedData(new BytesRef(internalNetworks), stringType, "internal_networks") + ), + "NetworkDirectionEvaluator[sourceIp=Attribute[channel=0], destinationIp=Attribute[channel=1]," + + " networks=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(expectedDirection)) + ) + ); + } + @Override protected Expression build(Source source, List args) { return new NetworkDirection(source, args.get(0), args.get(1), args.get(2)); From 8c400af31a752d5d9271117af22f1e23f20129ff Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 14 Oct 2025 14:10:08 +0000 Subject: [PATCH 23/31] [CI] Auto commit changes from spotless --- .../scalar/ip/NetworkDirectionTests.java | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 951a75303d944..0bffb711ec93c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -42,23 +42,52 @@ public static Iterable parameters() { createTestCase("CIDR1", "10.0.1.1", "192.168.1.2", "10.0.0.0/8", stringType, NetworkDirectionUtils.DIRECTION_OUTBOUND), createTestCase("CIDR2", "192.168.1.2", "10.0.1.1", "10.0.0.0/8", stringType, NetworkDirectionUtils.DIRECTION_INBOUND), // Unspecified tests - createTestCase("Unspecified1", "0.0.0.0", "0.0.0.0", "unspecified", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), + createTestCase( + "Unspecified1", + "0.0.0.0", + "0.0.0.0", + "unspecified", + stringType, + NetworkDirectionUtils.DIRECTION_INTERNAL + ), createTestCase("Unspecified2", "::", "::", "unspecified", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), // Private network tests createTestCase( - "Private1", "192.168.1.1", "192.168.1.2", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL + "Private1", + "192.168.1.1", + "192.168.1.2", + "private", + stringType, + NetworkDirectionUtils.DIRECTION_INTERNAL ), createTestCase("Private2", "10.0.1.1", "192.168.1.2", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), - createTestCase("Private3", "192.168.1.1", "172.16.0.1", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL), createTestCase( - "Private4", "192.168.1.1", "fd12:3456:789a:1::1", "private", stringType, NetworkDirectionUtils.DIRECTION_INTERNAL + "Private3", + "192.168.1.1", + "172.16.0.1", + "private", + stringType, + NetworkDirectionUtils.DIRECTION_INTERNAL + ), + createTestCase( + "Private4", + "192.168.1.1", + "fd12:3456:789a:1::1", + "private", + stringType, + NetworkDirectionUtils.DIRECTION_INTERNAL ), // Public tests createTestCase("Public1", "192.168.1.1", "192.168.1.2", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL), createTestCase("Public2", "10.0.1.1", "192.168.1.2", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL), createTestCase("Public3", "192.168.1.1", "172.16.0.1", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL), createTestCase( - "Public4", "192.168.1.1", "fd12:3456:789a:1::1", "public", stringType, NetworkDirectionUtils.DIRECTION_EXTERNAL + "Public4", + "192.168.1.1", + "fd12:3456:789a:1::1", + "public", + stringType, + NetworkDirectionUtils.DIRECTION_EXTERNAL ) ) ); From 925d79381bb022ea3a2e83a23c8349914271e62d Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 14 Oct 2025 10:15:00 -0400 Subject: [PATCH 24/31] Change caps for netdir func --- .../esql/qa/testFixtures/src/main/resources/ip.csv-spec | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index bf1f4e56fb6fd..21c327ccea7a2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -734,8 +734,8 @@ networkDirectionSimple required_capability: network_direction FROM hosts -| WHERE mv_count(ip0) == 1 AND mv_count(ip1) == 1 -| EVAL direction = network_direction(ip0, ip1, ["loopback"]) +| WHERE MV_COUNT(ip0) == 1 AND MV_COUNT(ip1) == 1 +| EVAL direction = NETWORK_DIRECTION(ip0, ip1, ["loopback"]) | KEEP ip0, ip1, direction | SORT direction, ip0, ip1 ; @@ -754,7 +754,7 @@ networkDirectionFromRow required_capability: network_direction ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] -| EVAL direction = network_direction(ip0, ip1, networks) +| EVAL direction = NETWORK_DIRECTION(ip0, ip1, networks) | DROP networks ; @@ -767,7 +767,7 @@ required_capability: network_direction // tag::networkDirectionFromRowWithInlineNetworks[] ROW ip0 = "127.0.0.1"::ip, ip1 = "5.6.7.8"::ip -| EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) +| EVAL direction = NETWORK_DIRECTION(ip0, ip1, ["loopback", "private"]) // end::networkDirectionFromRowWithInlineNetworks[] ; From c17a755ca4086646e87335e7af448a32b6e73a34 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 14 Oct 2025 11:10:18 -0400 Subject: [PATCH 25/31] Update NetworkDirection exception handling & add new CSV tests --- .../functions/examples/network_direction.md | 2 +- .../functions/network_direction.json | 2 +- .../docs/functions/network_direction.md | 2 +- .../src/main/resources/ip.csv-spec | 35 +++++++++++++++++++ .../scalar/ip/NetworkDirectionEvaluator.java | 7 +++- .../function/scalar/ip/NetworkDirection.java | 2 +- 6 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md index 8205925475594..761db02f26f94 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/network_direction.md @@ -4,7 +4,7 @@ ```esql ROW ip0 = "127.0.0.1"::ip, ip1 = "5.6.7.8"::ip -| EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) +| EVAL direction = NETWORK_DIRECTION(ip0, ip1, ["loopback", "private"]) ``` | ip0:ip | ip1:ip | direction:keyword | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json index 29910e16cd336..7427b1294499f 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json @@ -54,7 +54,7 @@ } ], "examples" : [ - "ROW ip0 = \"127.0.0.1\"::ip, ip1 = \"5.6.7.8\"::ip\n| EVAL direction = network_direction(ip0, ip1, [\"loopback\", \"private\"])" + "ROW ip0 = \"127.0.0.1\"::ip, ip1 = \"5.6.7.8\"::ip\n| EVAL direction = NETWORK_DIRECTION(ip0, ip1, [\"loopback\", \"private\"])" ], "preview" : false, "snapshot_only" : false diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md index 8746c2cc7e033..a70a9879df682 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/network_direction.md @@ -5,5 +5,5 @@ Returns the direction type (inbound, outbound, internal, external) given a sourc ```esql ROW ip0 = "127.0.0.1"::ip, ip1 = "5.6.7.8"::ip -| EVAL direction = network_direction(ip0, ip1, ["loopback", "private"]) +| EVAL direction = NETWORK_DIRECTION(ip0, ip1, ["loopback", "private"]) ``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index 21c327ccea7a2..11f37ab01faef 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -776,3 +776,38 @@ ip0:ip | ip1:ip | direction:keyword 127.0.0.1 | 5.6.7.8 | outbound // end::networkDirectionFromRowWithInlineNetworks-result[] ; + +networkDirectionFromIpPrefix +required_capability: network_direction + +ROW ip0 = "192.168.1.123"::ip, ip1 = "8.8.8.8"::ip +| EVAL direction = NETWORK_DIRECTION(IP_PREFIX(ip0, 24, 0), ip1, ["private"]) +| KEEP ip0, ip1, direction +; + +ip0:ip | ip1:ip | direction:keyword +192.168.1.123 | 8.8.8.8 | outbound +; + +networkDirectionBadIp +required_capability: network_direction + +ROW ip0 = null::ip, ip1 = "8.8.8.8"::ip +| EVAL direction = NETWORK_DIRECTION(TO_IP(ip0), ip1, ["loopback"]) +; + +ip0:ip | ip1:ip | direction:keyword +null | 8.8.8.8 | null +; + +networkDirectionBadNetworksArray +required_capability: network_direction + +ROW ip0 = "127.0.0.1"::ip, ip1 = "8.8.8.8"::ip +| EVAL direction = NETWORK_DIRECTION(ip0, ip1, ["invalid_network"]); +warning:Line 2:20: evaluation of [NETWORK_DIRECTION(ip0, ip1, [\"invalid_network\"])] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:20: java.lang.IllegalArgumentException: 'invalid_network' is not an IP string literal. + +ip0:ip |ip1:ip |direction:keyword +127.0.0.1 |8.8.8.8 |null +; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index 5dd8d477ba2b9..852afb3f42629 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -112,7 +112,12 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, } BytesRef sourceIp = sourceIpBlock.getBytesRef(sourceIpBlock.getFirstValueIndex(p), sourceIpScratch); BytesRef destinationIp = destinationIpBlock.getBytesRef(destinationIpBlock.getFirstValueIndex(p), destinationIpScratch); - NetworkDirection.process(result, this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock); + try { + NetworkDirection.process(result, this.scratch, this.netScratch, sourceIp, destinationIp, p, networksBlock); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index dd44f30256eac..d984f6c2b1be9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -132,7 +132,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua ); } - @Evaluator() + @Evaluator(warnExceptions = IllegalArgumentException.class) static void process( BytesRefBlock.Builder builder, @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef scratch, From dab39217b0a1ba06c0e1e7a0b12e8510493f3961 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 14 Oct 2025 11:12:17 -0400 Subject: [PATCH 26/31] Correct usage of forbidden APIs --- .../elasticsearch/common/network/NetworkDirectionUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java b/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java index 8aaba801daf0a..6ee23d3b7481d 100644 --- a/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java +++ b/server/src/main/java/org/elasticsearch/common/network/NetworkDirectionUtils.java @@ -53,9 +53,9 @@ public static boolean inNetwork(InetAddress address, String network) { case LINK_LOCAL_MULTICAST_NAMED_NETWORK -> isLinkLocalMulticast(address); case MULTICAST_NAMED_NETWORK -> isMulticast(address); case UNSPECIFIED_NAMED_NETWORK -> isUnspecified(address); - case PRIVATE_NAMED_NETWORK -> isPrivate(address.getHostAddress()); - case PUBLIC_NAMED_NETWORK -> isPublic(address.getHostAddress()); - default -> CIDRUtils.isInRange(address.getHostAddress(), network); + case PRIVATE_NAMED_NETWORK -> isPrivate(NetworkAddress.format(address)); + case PUBLIC_NAMED_NETWORK -> isPublic(NetworkAddress.format(address)); + default -> CIDRUtils.isInRange(NetworkAddress.format(address), network); }; } From 24d8f307acd7ed2e249774899a94cd0135464b16 Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 14 Oct 2025 11:21:34 -0400 Subject: [PATCH 27/31] Add netdir alias for network_direction ESQL function --- .../qa/testFixtures/src/main/resources/ip.csv-spec | 12 ++++++++++++ .../expression/function/EsqlFunctionRegistry.java | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index 11f37ab01faef..6f83b54606e05 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -762,6 +762,18 @@ ip0:ip | ip1:ip | direction:keyword 1.2.3.4 | 5.6.7.8 | external ; +networkDirectionFromRowUsingAlias +required_capability: network_direction + +ROW ip0 = "1.2.3.4"::ip, ip1 = "5.6.7.8"::ip, networks = ["loopback", "private"] +| EVAL direction = NETDIR(ip0, ip1, networks) +| DROP networks +; + +ip0:ip | ip1:ip | direction:keyword +1.2.3.4 | 5.6.7.8 | external +; + networkDirectionFromRowWithInlineNetworks required_capability: network_direction diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index f310aaa003cf6..3c3269feea37c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -454,7 +454,7 @@ private static FunctionDefinition[][] functions() { // IP new FunctionDefinition[] { def(CIDRMatch.class, CIDRMatch::new, "cidr_match") }, new FunctionDefinition[] { def(IpPrefix.class, IpPrefix::new, "ip_prefix") }, - new FunctionDefinition[] { def(NetworkDirection.class, NetworkDirection::new, "network_direction") }, + new FunctionDefinition[] { def(NetworkDirection.class, NetworkDirection::new, "network_direction", "netdir") }, // conversion functions new FunctionDefinition[] { def(FromBase64.class, FromBase64::new, "from_base64"), From 6a3486900229d78a597727df9a0ddaa94bbdd054 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 14 Oct 2025 15:43:10 +0000 Subject: [PATCH 28/31] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 1c09588ac7168..6f33c34d16b39 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1505,7 +1505,7 @@ public enum Cap { * Network direction function. */ NETWORK_DIRECTION(Build.current().isSnapshot()), - + /** * Support for the literal {@code m} suffix as an alias for {@code minute} in temporal amounts. */ From 18e39e64112c64430ced2a38616415eb7f0fe34e Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 14 Oct 2025 12:20:30 -0400 Subject: [PATCH 29/31] Stragglers --- .../scalar/ip/NetworkDirectionEvaluator.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java index 852afb3f42629..0dea21cab25d2 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionEvaluator.java @@ -81,27 +81,27 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock sourceIpBlock, BytesRef destinationIpScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { boolean allBlocksAreNulls = true; - if (sourceIpBlock.isNull(p)) { - result.appendNull(); - continue position; - } - if (sourceIpBlock.getValueCount(p) != 1) { - if (sourceIpBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; + switch (sourceIpBlock.getValueCount(p)) { + case 0: + result.appendNull(); + continue position; + case 1: + break; + default: + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + result.appendNull(); + continue position; } - if (destinationIpBlock.isNull(p)) { - result.appendNull(); - continue position; - } - if (destinationIpBlock.getValueCount(p) != 1) { - if (destinationIpBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; + switch (destinationIpBlock.getValueCount(p)) { + case 0: + result.appendNull(); + continue position; + case 1: + break; + default: + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + result.appendNull(); + continue position; } if (!networksBlock.isNull(p)) { allBlocksAreNulls = false; From ec37f6496d4840ad251e2ac239d70463a5cd95af Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Tue, 21 Oct 2025 13:13:42 -0400 Subject: [PATCH 30/31] Specify that function is in preview mode --- .../esql/kibana/definition/functions/network_direction.json | 2 +- .../esql/expression/function/scalar/ip/NetworkDirection.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json index 7427b1294499f..f921b8e32822a 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/network_direction.json @@ -56,6 +56,6 @@ "examples" : [ "ROW ip0 = \"127.0.0.1\"::ip, ip1 = \"5.6.7.8\"::ip\n| EVAL direction = NETWORK_DIRECTION(ip0, ip1, [\"loopback\", \"private\"])" ], - "preview" : false, + "preview" : true, "snapshot_only" : false } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java index d984f6c2b1be9..5f16967a398c5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirection.java @@ -57,6 +57,7 @@ public class NetworkDirection extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", + preview = true, description = "Returns the direction type (inbound, outbound, internal, external) given " + "a source IP address, destination IP address, and a list of internal networks.", examples = @Example(file = "ip", tag = "networkDirectionFromRowWithInlineNetworks") From 756fc5ac5b9489a2984891c112d74f00aa8e0b0c Mon Sep 17 00:00:00 2001 From: Matthew Alp Date: Wed, 22 Oct 2025 12:00:23 -0400 Subject: [PATCH 31/31] Change import --- .../expression/function/scalar/ip/NetworkDirectionTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java index 0bffb711ec93c..3c6405360a5df 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/NetworkDirectionTests.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.Matchers.equalTo; public class NetworkDirectionTests extends AbstractScalarFunctionTestCase { public NetworkDirectionTests(@Name("TestCase") Supplier testCaseSupplier) {