From 69cfa82fa2d388ce1c1fcbd1f74aeef9674e6b75 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Tue, 19 Aug 2025 01:44:19 +0200 Subject: [PATCH 01/41] Add `mv_contains_all` function --- .../functions/description/mv_contains_all.md | 6 + .../functions/examples/mv_contains_all.md | 23 ++ .../functions/layout/mv_contains_all.md | 23 ++ .../functions/parameters/mv_contains_all.md | 10 + .../functions/types/mv_contains_all.md | 24 ++ .../esql/images/functions/mv_contains_all.svg | 1 + .../definition/functions/mv_contains_all.json | 320 ++++++++++++++++++ .../kibana/docs/functions/mv_contains_all.md | 9 + .../src/main/resources/string.csv-spec | 41 +++ .../MvContainsAllBooleanEvaluator.java | 116 +++++++ .../MvContainsAllBytesRefEvaluator.java | 118 +++++++ .../MvContainsAllDoubleEvaluator.java | 117 +++++++ .../multivalue/MvContainsAllIntEvaluator.java | 117 +++++++ .../MvContainsAllLongEvaluator.java | 117 +++++++ .../function/EsqlFunctionRegistry.java | 2 + .../scalar/multivalue/MvContainsAll.java | 273 +++++++++++++++ .../multivalue/MvFunctionWritables.java | 1 + .../multivalue/MvContainsAllErrorTests.java | 45 +++ .../MvContainsAllSerializationTests.java | 37 ++ .../scalar/multivalue/MvContainsAllTests.java | 273 +++++++++++++++ 20 files changed, 1673 insertions(+) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains_all.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/mv_contains_all.md create mode 100644 docs/reference/query-languages/esql/images/functions/mv_contains_all.svg create mode 100644 docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json create mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md new file mode 100644 index 0000000000000..dd43a5b97309b --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Checks if the values yielded by multivalue value expression are all also present in the values yielded by anothermultivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md new file mode 100644 index 0000000000000..1da12b2fc7645 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md @@ -0,0 +1,23 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Examples** + +```esql +ROW a = "a", b = ["a", "b", "c"] +| EVAL a_subset_of_b = mv_contains_all(b, a) +``` + +| a:keyword | b:keyword | a_subset_of_b:boolean | +| --- | --- | --- | +| a | [a, b, c] | true | + +```esql +ROW a = ["a","c"], b = ["a", "b", "c"] +| EVAL a_subset_of_b = mv_contains_all(b, a) +``` + +| a:keyword | b:keyword | a_subset_of_b:boolean | +| --- | --- | --- | +| [a, c] | [a, b, c] | true | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md new file mode 100644 index 0000000000000..42bc9c85f0ebd --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md @@ -0,0 +1,23 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `MV_CONTAINS_ALL` [esql-mv_contains_all] + +**Syntax** + +:::{image} ../../../images/functions/mv_contains_all.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/mv_contains_all.md +::: + +:::{include} ../description/mv_contains_all.md +::: + +:::{include} ../types/mv_contains_all.md +::: + +:::{include} ../examples/mv_contains_all.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains_all.md new file mode 100644 index 0000000000000..9f002719dbd99 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains_all.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`superset` +: Multivalue expression. + +`subset` +: Multivalue expression. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/types/mv_contains_all.md new file mode 100644 index 0000000000000..fe4b46ac5b280 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/mv_contains_all.md @@ -0,0 +1,24 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| superset | subset | result | +| --- | --- | --- | +| boolean | boolean | boolean | +| cartesian_point | cartesian_point | boolean | +| cartesian_shape | cartesian_shape | boolean | +| date | date | boolean | +| date_nanos | date_nanos | boolean | +| double | double | boolean | +| geo_point | geo_point | boolean | +| geo_shape | geo_shape | boolean | +| integer | integer | boolean | +| ip | ip | boolean | +| keyword | keyword | boolean | +| keyword | text | boolean | +| long | long | boolean | +| text | keyword | boolean | +| text | text | boolean | +| unsigned_long | unsigned_long | boolean | +| version | version | boolean | + diff --git a/docs/reference/query-languages/esql/images/functions/mv_contains_all.svg b/docs/reference/query-languages/esql/images/functions/mv_contains_all.svg new file mode 100644 index 0000000000000..f7e5b3807492b --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/mv_contains_all.svg @@ -0,0 +1 @@ +MV_CONTAINS_ALL(superset,subset) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json new file mode 100644 index 0000000000000..f8677e62cd44a --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json @@ -0,0 +1,320 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "scalar", + "name" : "mv_contains_all", + "description" : "Checks if the values yielded by multivalue value expression are all also present in the values yielded by anothermultivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", + "signatures" : [ + { + "params" : [ + { + "name" : "superset", + "type" : "boolean", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "boolean", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "cartesian_point", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "cartesian_point", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "date", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "date", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "double", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "double", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "geo_point", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "geo_point", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "geo_shape", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "geo_shape", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "integer", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "integer", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "ip", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "ip", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "keyword", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "keyword", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "keyword", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "text", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "long", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "long", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "text", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "keyword", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "text", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "text", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "unsigned_long", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "unsigned_long", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "superset", + "type" : "version", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "subset", + "type" : "version", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "ROW a = \"a\", b = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains_all(b, a)", + "ROW a = [\"a\",\"c\"], b = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains_all(b, a)" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md b/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md new file mode 100644 index 0000000000000..4d22c719516d4 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### MV CONTAINS ALL +Checks if the values yielded by multivalue value expression are all also present in the values yielded by anothermultivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. + +```esql +ROW a = "a", b = ["a", "b", "c"] +| EVAL a_subset_of_b = mv_contains_all(b, a) +``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 2602378c64615..51639afb370d1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1931,6 +1931,47 @@ l1:integer | l2:integer null | 0 ; +mvAny#[skip:-9.2.99,reason:new base64 function added in 9.3] +// tag::mv_contains_all[] +ROW a = "a", b = ["a", "b", "c"] +| EVAL a_subset_of_b = mv_contains_all(b, a) +// end::mv_contains_all[] +; + +// tag::mv_contains_all-result[] +a:keyword | b:keyword | a_subset_of_b:boolean +a | [a, b, c] | true +// end::mv_contains_all-result[] +; + +mvAny_bothsides#[skip:-9.2.99,reason:new base64 function added in 9.3] +// tag::mv_contains_all_bothsides[] +ROW a = ["a","c"], b = ["a", "b", "c"] +| EVAL a_subset_of_b = mv_contains_all(b, a) +// end::mv_contains_all_bothsides[] +; + +// tag::mv_contains_all_bothsides-result[] +a:keyword | b:keyword | a_subset_of_b:boolean +[a, c] | [a, b, c] | true +// end::mv_contains_all_bothsides-result[] +; + +mvAnyCombinations#[skip:-9.2.99,reason:new base64 function added in 9.3] + +ROW a = "a", b = ["a", "b", "c"], n = null +| EVAL aa = mv_contains_all(a, a), + bb = mv_contains_all(b, b), + ab = mv_contains_all(a, b), + ba = mv_contains_all(b,a), + na = mv_contains_all(n, a), + an = mv_contains_all(a, n), + nn = mv_contains_all(n,n) +; + +a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:boolean +a | [a, b, c] | null | true | true | false | true | null | null | null +; mvAppend required_capability: fn_mv_append diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java new file mode 100644 index 0000000000000..4eb973c496eaf --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java @@ -0,0 +1,116 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. + * This class is generated. Edit {@code EvaluatorImplementer} instead. + */ +public final class MvContainsAllBooleanEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + private Warnings warnings; + + public MvContainsAllBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BooleanBlock field1Block = (BooleanBlock) field1.eval(page)) { + try (BooleanBlock field2Block = (BooleanBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvContainsAll.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsAllBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsAllBooleanEvaluator get(DriverContext context) { + return new MvContainsAllBooleanEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsAllBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java new file mode 100644 index 0000000000000..8d345c367fb39 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java @@ -0,0 +1,118 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. + * This class is generated. Edit {@code EvaluatorImplementer} instead. + */ +public final class MvContainsAllBytesRefEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + private Warnings warnings; + + public MvContainsAllBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock field1Block = (BytesRefBlock) field1.eval(page)) { + try (BytesRefBlock field2Block = (BytesRefBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, BytesRefBlock field1Block, + BytesRefBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvContainsAll.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsAllBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsAllBytesRefEvaluator get(DriverContext context) { + return new MvContainsAllBytesRefEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsAllBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java new file mode 100644 index 0000000000000..9cf90bdc3786b --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java @@ -0,0 +1,117 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. + * This class is generated. Edit {@code EvaluatorImplementer} instead. + */ +public final class MvContainsAllDoubleEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + private Warnings warnings; + + public MvContainsAllDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (DoubleBlock field1Block = (DoubleBlock) field1.eval(page)) { + try (DoubleBlock field2Block = (DoubleBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvContainsAll.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsAllDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsAllDoubleEvaluator get(DriverContext context) { + return new MvContainsAllDoubleEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsAllDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java new file mode 100644 index 0000000000000..0d5a41f992d91 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java @@ -0,0 +1,117 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. + * This class is generated. Edit {@code EvaluatorImplementer} instead. + */ +public final class MvContainsAllIntEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + private Warnings warnings; + + public MvContainsAllIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (IntBlock field1Block = (IntBlock) field1.eval(page)) { + try (IntBlock field2Block = (IntBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvContainsAll.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsAllIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsAllIntEvaluator get(DriverContext context) { + return new MvContainsAllIntEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsAllIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java new file mode 100644 index 0000000000000..2e7cb847f52ce --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java @@ -0,0 +1,117 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. + * This class is generated. Edit {@code EvaluatorImplementer} instead. + */ +public final class MvContainsAllLongEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + private Warnings warnings; + + public MvContainsAllLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock field1Block = (LongBlock) field1.eval(page)) { + try (LongBlock field2Block = (LongBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock field1Block, LongBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvContainsAll.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsAllLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsAllLongEvaluator get(DriverContext context) { + return new MvContainsAllLongEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsAllLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 0202661acf76d..2238342ed985e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -120,6 +120,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tan; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tanh; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; +import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvContainsAll; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat; @@ -449,6 +450,7 @@ private static FunctionDefinition[][] functions() { def(MvAppend.class, MvAppend::new, "mv_append"), def(MvAvg.class, MvAvg::new, "mv_avg"), def(MvConcat.class, MvConcat::new, "mv_concat"), + def(MvContainsAll.class, MvContainsAll::new, "mv_contains_all"), def(MvCount.class, MvCount::new, "mv_count"), def(MvDedupe.class, MvDedupe::new, "mv_dedupe"), def(MvFirst.class, MvFirst::new, "mv_first"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java new file mode 100644 index 0000000000000..5b1678f0924b2 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -0,0 +1,273 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; + +import java.io.IOException; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; + +/** + * Reduce a multivalued field to a single valued field containing the count of values. + */ +public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapper { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvContainsAll", MvContainsAll::new); + private DataType dataType; + + @FunctionInfo( + returnType = "boolean", + description = "Checks if the values yielded by multivalue value expression are all also present in the values yielded by another" + + "multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", + examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), } + ) + public MvContainsAll( + Source source, + @Param( + name = "superset", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "Multivalue expression." + ) Expression superset, + @Param( + name = "subset", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "Multivalue expression." + ) Expression subset + ) { + super(source, superset, subset); + } + + private MvContainsAll(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isRepresentableExceptCounters(left(), sourceText(), FIRST); + if (resolution.unresolved()) { + return resolution; + } + dataType = left().dataType() == DataType.NULL ? DataType.NULL : DataType.BOOLEAN; + if (left().dataType() == DataType.NULL) { + dataType = right().dataType() == DataType.NULL ? DataType.NULL : DataType.BOOLEAN; + return isRepresentableExceptCounters(right(), sourceText(), SECOND); + } + return isType(right(), t -> t.noText() == left().dataType().noText(), sourceText(), SECOND, left().dataType().noText().typeName()); + } + + @Override + public DataType dataType() { + if (dataType == null) { + resolveType(); + } + return dataType; + } + + @Override + protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) { + return new MvContainsAll(source(), newLeft, newRight); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, MvContainsAll::new, left(), right()); + } + + @Override + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); + } + + @Override + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var supersetType = PlannerUtils.toElementType(left().dataType()); + var subsetType = PlannerUtils.toElementType(right().dataType()); + if( supersetType != subsetType ) { + throw new EsqlIllegalArgumentException( + "Incompatible data types for MvContainsAll, superset type({}) value({}) and subset type({}) value({}) don't match.", + supersetType, left(), + subsetType, right() + ); + } + return switch (supersetType) { + case BOOLEAN -> new MvContainsAllBooleanEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case BYTES_REF -> new MvContainsAllBytesRefEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case DOUBLE -> new MvContainsAllDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case INT -> new MvContainsAllIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case LONG -> new MvContainsAllLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; + default -> throw EsqlIllegalArgumentException.illegalDataType(dataType()); + }; + } + + @Evaluator(extraName = "Int") + static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { + appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt)); + } + + @Evaluator(extraName = "Boolean") + static void process(BooleanBlock.Builder builder, int position, BooleanBlock field1, BooleanBlock field2) { + appendTo(builder, containsAll(field1, field2, position, BooleanBlock::getBoolean)); + } + + @Evaluator(extraName = "Long") + static void process(BooleanBlock.Builder builder, int position, LongBlock field1, LongBlock field2) { + appendTo(builder, containsAll(field1, field2, position, LongBlock::getLong)); + } + + @Evaluator(extraName = "Double") + static void process(BooleanBlock.Builder builder, int position, DoubleBlock field1, DoubleBlock field2) { + appendTo(builder, containsAll(field1, field2, position, DoubleBlock::getDouble)); + } + + @Evaluator(extraName = "BytesRef") + static void process(BooleanBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) { + appendTo( + builder, + containsAll(field1, field2, position, (block, index) -> { + var ref = new BytesRef(); + block.getBytesRef(index, ref); + return ref; + }) + ); + } + + static void appendTo(BooleanBlock.Builder builder, Boolean bool) { + if(bool == null) { + builder.appendNull(); + } else { + builder.beginPositionEntry().appendBoolean(bool).endPositionEntry(); + } + } + + /** + * A block is considered a subset if the superset contains values that test equal for all the values in the subset, independent of + * order. Duplicates are ignored in the sense that for each duplicate in the subset, we will search/match against the first/any value + * in the superset. + * @param superset block to check against + * @param subset block containing values that should be present in the other block. + * @return {@code true} if the given blocks are a superset and subset to each other, {@code false} if not and {@code null} if the subset + * or superset contains only null values. + */ + static Boolean containsAll( + BlockType superset, + BlockType subset, + final int position, + ValueExtractor valueExtractor + ) { + if (superset == subset) { + return true; + } + if (subset.areAllValuesNull() || superset.areAllValuesNull()) { + return null; + } + + final var subsetCount = subset.getValueCount(position); + final var startIndex = subset.getFirstValueIndex(position); + for (int subsetIndex = startIndex; subsetIndex < startIndex+subsetCount; subsetIndex++) { + var value = valueExtractor.extractValue(subset, subsetIndex); + if (hasValue(superset, position, value, valueExtractor) == false) { + return false; + } + } + return true; + } + + /** + * Check if the block has the value at any of it's positions + * @param superset Block to search + * @param value to search for + * @return true if the supplied long value is in the supplied Block + */ + static boolean hasValue( + BlockType superset, + final int position, + Type value, + ValueExtractor valueExtractor + ) { + final var supersetCount = superset.getValueCount(position); + final var startIndex = superset.getFirstValueIndex(position); + for (int supersetIndex = startIndex; supersetIndex < startIndex+supersetCount; supersetIndex++) { + var element = valueExtractor.extractValue(superset, supersetIndex); + if(element.equals(value)) { + return true; + } + } + return false; + } + + interface ValueExtractor { + Type extractValue(BlockType block, int position); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java index 7f8fcd910ad6d..680542fc8e462 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java @@ -17,6 +17,7 @@ public static List getNamedWriteables() { MvAppend.ENTRY, MvAvg.ENTRY, MvConcat.ENTRY, + MvContainsAll.ENTRY, MvCount.ENTRY, MvDedupe.ENTRY, MvFirst.ENTRY, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllErrorTests.java new file mode 100644 index 0000000000000..4c6bafdf8ed84 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllErrorTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class MvContainsAllErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(MvContainsAllTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new MvContainsAll(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + "second argument of [" + + sourceForSignature(signature) + + "] must be [" + + signature.get(0).noText().typeName() + + "], found value [] type [" + + signature.get(1).typeName() + + "]" + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllSerializationTests.java new file mode 100644 index 0000000000000..8663d3cc9c574 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvContainsAllSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected MvContainsAll createTestInstance() { + Source source = randomSource(); + Expression field1 = randomChild(); + Expression field2 = randomChild(); + return new MvContainsAll(source, field1, field2); + } + + @Override + protected MvContainsAll mutateInstance(MvContainsAll instance) throws IOException { + Source source = randomSource(); + Expression field1 = randomChild(); + Expression field2 = randomChild(); + if (randomBoolean()) { + field1 = randomValueOtherThan(field1, AbstractExpressionSerializationTests::randomChild); + } else { + field2 = randomValueOtherThan(field2, AbstractExpressionSerializationTests::randomChild); + } + return new MvContainsAll(source, field1, field2); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java new file mode 100644 index 0000000000000..59a6f7dc08290 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java @@ -0,0 +1,273 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; +import static org.hamcrest.Matchers.equalTo; + +public class MvContainsAllTests extends AbstractScalarFunctionTestCase { + public MvContainsAllTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + booleans(suppliers); + ints(suppliers); + longs(suppliers); + doubles(suppliers); + bytesRefs(suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new MvContainsAll(source, args.get(0), args.get(1)); + } + + private static void booleans(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataType.BOOLEAN, DataType.BOOLEAN), () -> { + List field1 = randomList(1, 10, ESTestCase::randomBoolean); + List field2 = randomList(1, 2, ESTestCase::randomBoolean); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.BOOLEAN, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.BOOLEAN, "field2") + ), + "MvContainsAllBooleanEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + } + + private static void ints(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataType.INTEGER, DataType.INTEGER), () -> { + List field1 = randomList(1, 10, ESTestCase::randomInt); + List field2 = randomList(1, 10, ESTestCase::randomInt); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.INTEGER, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.INTEGER, "field2") + ), + "MvContainsAllIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + } + + private static void longs(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataType.LONG, DataType.LONG), () -> { + List field1 = randomList(1, 10, ESTestCase::randomLong); + List field2 = randomList(1, 10, ESTestCase::randomLong); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.LONG, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.LONG, "field2") + ), + "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.UNSIGNED_LONG, DataType.UNSIGNED_LONG), () -> { + List field1 = randomList(1, 10, ESTestCase::randomLong); + List field2 = randomList(1, 10, ESTestCase::randomLong); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.UNSIGNED_LONG, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.UNSIGNED_LONG, "field2") + ), + "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.DATETIME, DataType.DATETIME), () -> { + List field1 = randomList(1, 10, ESTestCase::randomLong); + List field2 = randomList(1, 10, ESTestCase::randomLong); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.DATETIME, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.DATETIME, "field2") + ), + "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.DATE_NANOS, DataType.DATE_NANOS), () -> { + List field1 = randomList(1, 10, ESTestCase::randomNonNegativeLong); + List field2 = randomList(1, 10, ESTestCase::randomNonNegativeLong); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.DATE_NANOS, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.DATE_NANOS, "field2") + ), + "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + } + + private static void doubles(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataType.DOUBLE, DataType.DOUBLE), () -> { + List field1 = randomList(1, 10, ESTestCase::randomDouble); + List field2 = randomList(1, 10, ESTestCase::randomDouble); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.DOUBLE, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.DOUBLE, "field2") + ), + "MvContainsAllDoubleEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + } + + private static void bytesRefs(List suppliers) { + for (DataType lhs : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType rhs : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + suppliers.add(new TestCaseSupplier(List.of(lhs, rhs), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(lhs).value()); + List field2 = randomList(1, 10, () -> randomLiteral(rhs).value()); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, lhs, "field1"), + new TestCaseSupplier.TypedData(field2, rhs, "field2") + ), + "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + } + } + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.IP, "field"), + new TestCaseSupplier.TypedData(field2, DataType.IP, "field") + ), + "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.VERSION), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value()); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.VERSION, "field"), + new TestCaseSupplier.TypedData(field2, DataType.VERSION, "field") + ), + "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_POINT, DataType.GEO_POINT), () -> { + List field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint()))); + List field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint()))); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.GEO_POINT, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.GEO_POINT, "field2") + ), + "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_POINT, DataType.CARTESIAN_POINT), () -> { + List field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint()))); + List field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint()))); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_POINT, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_POINT, "field2") + ), + "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_SHAPE, DataType.GEO_SHAPE), () -> { + var field1 = randomList(1, 3, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean(), 500)))); + var field2 = randomList(1, 3, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean(), 500)))); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.GEO_SHAPE, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.GEO_SHAPE, "field2") + ), + "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_SHAPE, DataType.CARTESIAN_SHAPE), () -> { + var field1 = randomList(1, 3, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean(), 500)))); + var field2 = randomList(1, 3, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean(), 500)))); + var result = field1.containsAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_SHAPE, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_SHAPE, "field2") + ), + "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(result) + ); + })); + } + +} From e25024788ccf667af18b70e8d3189fe0f0bf95c0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 19 Aug 2025 00:05:06 +0000 Subject: [PATCH 02/41] [CI] Auto commit changes from spotless --- .../function/EsqlFunctionRegistry.java | 2 +- .../scalar/multivalue/MvContainsAll.java | 40 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 2238342ed985e..cfc8443617e6d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -120,10 +120,10 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tan; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tanh; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvContainsAll; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat; +import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvContainsAll; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvDedupe; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvFirst; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index 5b1678f0924b2..000ec22107b94 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -43,13 +43,17 @@ * Reduce a multivalued field to a single valued field containing the count of values. */ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapper { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvContainsAll", MvContainsAll::new); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "MvContainsAll", + MvContainsAll::new + ); private DataType dataType; @FunctionInfo( returnType = "boolean", - description = "Checks if the values yielded by multivalue value expression are all also present in the values yielded by another" + - "multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", + description = "Checks if the values yielded by multivalue value expression are all also present in the values yielded by another" + + "multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), } ) public MvContainsAll( @@ -107,7 +111,6 @@ public String getWriteableName() { return ENTRY.name; } - @Override protected TypeResolution resolveType() { if (childrenResolved() == false) { @@ -153,11 +156,13 @@ public Object fold(FoldContext ctx) { public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var supersetType = PlannerUtils.toElementType(left().dataType()); var subsetType = PlannerUtils.toElementType(right().dataType()); - if( supersetType != subsetType ) { + if (supersetType != subsetType) { throw new EsqlIllegalArgumentException( "Incompatible data types for MvContainsAll, superset type({}) value({}) and subset type({}) value({}) don't match.", - supersetType, left(), - subsetType, right() + supersetType, + left(), + subsetType, + right() ); } return switch (supersetType) { @@ -193,18 +198,15 @@ static void process(BooleanBlock.Builder builder, int position, DoubleBlock fiel @Evaluator(extraName = "BytesRef") static void process(BooleanBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) { - appendTo( - builder, - containsAll(field1, field2, position, (block, index) -> { - var ref = new BytesRef(); - block.getBytesRef(index, ref); - return ref; - }) - ); + appendTo(builder, containsAll(field1, field2, position, (block, index) -> { + var ref = new BytesRef(); + block.getBytesRef(index, ref); + return ref; + })); } static void appendTo(BooleanBlock.Builder builder, Boolean bool) { - if(bool == null) { + if (bool == null) { builder.appendNull(); } else { builder.beginPositionEntry().appendBoolean(bool).endPositionEntry(); @@ -235,7 +237,7 @@ static Boolean containsAll( final var subsetCount = subset.getValueCount(position); final var startIndex = subset.getFirstValueIndex(position); - for (int subsetIndex = startIndex; subsetIndex < startIndex+subsetCount; subsetIndex++) { + for (int subsetIndex = startIndex; subsetIndex < startIndex + subsetCount; subsetIndex++) { var value = valueExtractor.extractValue(subset, subsetIndex); if (hasValue(superset, position, value, valueExtractor) == false) { return false; @@ -258,9 +260,9 @@ static boolean hasValue( ) { final var supersetCount = superset.getValueCount(position); final var startIndex = superset.getFirstValueIndex(position); - for (int supersetIndex = startIndex; supersetIndex < startIndex+supersetCount; supersetIndex++) { + for (int supersetIndex = startIndex; supersetIndex < startIndex + supersetCount; supersetIndex++) { var element = valueExtractor.extractValue(superset, supersetIndex); - if(element.equals(value)) { + if (element.equals(value)) { return true; } } From f02c4000820da7ae711ec3e05f76d7d4dfbf1fb7 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Tue, 19 Aug 2025 03:50:37 +0200 Subject: [PATCH 03/41] Enhance `mv_contains_all` function with additional examples and documentation improvements. Fix issue with byteref being empty, which caused fold to fail. --- .../functions/examples/mv_contains_all.md | 10 ++++++++++ .../definition/functions/mv_contains_all.json | 3 ++- .../src/main/resources/string.csv-spec | 20 ++++++++++++++++--- .../scalar/multivalue/MvContainsAll.java | 12 ++++++++--- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md index 1da12b2fc7645..d0e290259e3d4 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md @@ -20,4 +20,14 @@ ROW a = ["a","c"], b = ["a", "b", "c"] | --- | --- | --- | | [a, c] | [a, b, c] | true | +```esql +FROM airports +| WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9 +| KEEP scalerank, name, country +``` + +| scalerank:integer | name:text | country:keyword | +| --- | --- | --- | +| 8 | Chandigarh Int'l | India | + diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json index f8677e62cd44a..28e65a61c8843 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json @@ -313,7 +313,8 @@ ], "examples" : [ "ROW a = \"a\", b = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains_all(b, a)", - "ROW a = [\"a\",\"c\"], b = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains_all(b, a)" + "ROW a = [\"a\",\"c\"], b = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains_all(b, a)", + "FROM airports\n| WHERE mv_contains_all(type, [\"major\",\"military\"]) AND scalerank == 9\n| KEEP scalerank, name, country" ], "preview" : false, "snapshot_only" : false diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 51639afb370d1..d8d9d62ec57bb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1931,7 +1931,7 @@ l1:integer | l2:integer null | 0 ; -mvAny#[skip:-9.2.99,reason:new base64 function added in 9.3] +mvContainsAll#[skip:-9.2.99,reason:new base64 function added in 9.3] // tag::mv_contains_all[] ROW a = "a", b = ["a", "b", "c"] | EVAL a_subset_of_b = mv_contains_all(b, a) @@ -1944,7 +1944,7 @@ a | [a, b, c] | true // end::mv_contains_all-result[] ; -mvAny_bothsides#[skip:-9.2.99,reason:new base64 function added in 9.3] +mvContainsAll_bothsides#[skip:-9.2.99,reason:new base64 function added in 9.3] // tag::mv_contains_all_bothsides[] ROW a = ["a","c"], b = ["a", "b", "c"] | EVAL a_subset_of_b = mv_contains_all(b, a) @@ -1957,7 +1957,7 @@ a:keyword | b:keyword | a_subset_of_b:boolean // end::mv_contains_all_bothsides-result[] ; -mvAnyCombinations#[skip:-9.2.99,reason:new base64 function added in 9.3] +mvContainsAllCombinations#[skip:-9.2.99,reason:new base64 function added in 9.3] ROW a = "a", b = ["a", "b", "c"], n = null | EVAL aa = mv_contains_all(a, a), @@ -1973,6 +1973,20 @@ a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boo a | [a, b, c] | null | true | true | false | true | null | null | null ; +mvContainsAll_where#[skip:-9.2.99,reason:new base64 function added in 9.3] +// tag::mv_contains_all_where[] +FROM airports +| WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9 +| KEEP scalerank, name, country +// end::mv_contains_all_where[] +; + +// tag::mv_contains_all_where-result[] +scalerank:integer | name:text | country:keyword +8 | Chandigarh Int'l | India +// end::mv_contains_all_where-result[] +; + mvAppend required_capability: fn_mv_append diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index 000ec22107b94..2dedcac328d92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -54,7 +54,8 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp returnType = "boolean", description = "Checks if the values yielded by multivalue value expression are all also present in the values yielded by another" + "multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", - examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), } + examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), + @Example(file = "string", tag = "mv_contains_all_where"),} ) public MvContainsAll( Source source, @@ -200,7 +201,12 @@ static void process(BooleanBlock.Builder builder, int position, DoubleBlock fiel static void process(BooleanBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) { appendTo(builder, containsAll(field1, field2, position, (block, index) -> { var ref = new BytesRef(); - block.getBytesRef(index, ref); + // we pass in a reference, but sometimes we only get a return value, see ConstantBytesRefVector.getBytesRef + ref = block.getBytesRef(index, ref); + // pass empty ref as null + if(ref.length == 0) { + return null; + } return ref; })); } @@ -262,7 +268,7 @@ static boolean hasValue( final var startIndex = superset.getFirstValueIndex(position); for (int supersetIndex = startIndex; supersetIndex < startIndex + supersetCount; supersetIndex++) { var element = valueExtractor.extractValue(superset, supersetIndex); - if (element.equals(value)) { + if (element != null && element.equals(value)) { return true; } } From 4d0d5bc32dab09eeaf3d571f5838468cd0f41671 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 19 Aug 2025 02:00:27 +0000 Subject: [PATCH 04/41] [CI] Auto commit changes from spotless --- .../function/scalar/multivalue/MvContainsAll.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index 2dedcac328d92..b990fe8288a7a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -54,8 +54,10 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp returnType = "boolean", description = "Checks if the values yielded by multivalue value expression are all also present in the values yielded by another" + "multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", - examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), - @Example(file = "string", tag = "mv_contains_all_where"),} + examples = { + @Example(file = "string", tag = "mv_contains_all"), + @Example(file = "string", tag = "mv_contains_all_bothsides"), + @Example(file = "string", tag = "mv_contains_all_where"), } ) public MvContainsAll( Source source, @@ -204,7 +206,7 @@ static void process(BooleanBlock.Builder builder, int position, BytesRefBlock fi // we pass in a reference, but sometimes we only get a return value, see ConstantBytesRefVector.getBytesRef ref = block.getBytesRef(index, ref); // pass empty ref as null - if(ref.length == 0) { + if (ref.length == 0) { return null; } return ref; From dd44baae2d1b29532f148fc39cc8d12741299dca Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Tue, 19 Aug 2025 04:03:18 +0200 Subject: [PATCH 05/41] Add documentation for `mv_contains_all` function --- .../query-languages/esql/_snippets/lists/mv-functions.md | 1 + .../query-languages/esql/functions-operators/mv-functions.md | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md b/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md index a7b32dfb3835e..983d886c3568f 100644 --- a/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md +++ b/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md @@ -1,6 +1,7 @@ * [`MV_APPEND`](../../functions-operators/mv-functions.md#esql-mv_append) * [`MV_AVG`](../../functions-operators/mv-functions.md#esql-mv_avg) * [`MV_CONCAT`](../../functions-operators/mv-functions.md#esql-mv_concat) +* [`MV_CONTAINS_ALL`](../../functions-operators/mv-functions.md#esql-mv_contains_all) * [`MV_COUNT`](../../functions-operators/mv-functions.md#esql-mv_count) * [`MV_DEDUPE`](../../functions-operators/mv-functions.md#esql-mv_dedupe) * [`MV_FIRST`](../../functions-operators/mv-functions.md#esql-mv_first) diff --git a/docs/reference/query-languages/esql/functions-operators/mv-functions.md b/docs/reference/query-languages/esql/functions-operators/mv-functions.md index 7eca1a53ab8ff..81c542b9ade1d 100644 --- a/docs/reference/query-languages/esql/functions-operators/mv-functions.md +++ b/docs/reference/query-languages/esql/functions-operators/mv-functions.md @@ -21,6 +21,9 @@ mapped_pages: :::{include} ../_snippets/functions/layout/mv_concat.md ::: +:::{include} ../_snippets/functions/layout/mv_contains_all.md +::: + :::{include} ../_snippets/functions/layout/mv_count.md ::: From fa3a3396d1cc3f985d2eb3c2b42a291d9b2ca366 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Tue, 19 Aug 2025 04:08:43 +0200 Subject: [PATCH 06/41] Fixing documentation for `mv_contains_all` function --- .../esql/qa/testFixtures/src/main/resources/string.csv-spec | 2 +- .../expression/function/scalar/multivalue/MvContainsAll.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index d8d9d62ec57bb..c4562fa62dc25 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1983,7 +1983,7 @@ FROM airports // tag::mv_contains_all_where-result[] scalerank:integer | name:text | country:keyword -8 | Chandigarh Int'l | India +9 | Chandigarh Int'l | India // end::mv_contains_all_where-result[] ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index b990fe8288a7a..cdf7065aea861 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -52,7 +52,7 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp @FunctionInfo( returnType = "boolean", - description = "Checks if the values yielded by multivalue value expression are all also present in the values yielded by another" + description = "Checks if the values yielded by multivalue value expression are all present in the values yielded by the other " + "multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", examples = { @Example(file = "string", tag = "mv_contains_all"), From 05d64b6cebe917c7b1fc1c02a87d469baed97e74 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Tue, 19 Aug 2025 04:12:03 +0200 Subject: [PATCH 07/41] Regenerated `mv_contains_all` documentation --- .../esql/_snippets/functions/description/mv_contains_all.md | 2 +- .../esql/_snippets/functions/examples/mv_contains_all.md | 2 +- .../esql/kibana/definition/functions/mv_contains_all.json | 2 +- .../esql/kibana/docs/functions/mv_contains_all.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md index dd43a5b97309b..f08a0913b77e5 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md +++ b/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md @@ -2,5 +2,5 @@ **Description** -Checks if the values yielded by multivalue value expression are all also present in the values yielded by anothermultivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. +Checks if the values yielded by multivalue value expression are all present in the values yielded by the other multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md index d0e290259e3d4..aaf2ec00ee7cb 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md @@ -28,6 +28,6 @@ FROM airports | scalerank:integer | name:text | country:keyword | | --- | --- | --- | -| 8 | Chandigarh Int'l | India | +| 9 | Chandigarh Int'l | India | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json index 28e65a61c8843..aba4c900335ca 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", "type" : "scalar", "name" : "mv_contains_all", - "description" : "Checks if the values yielded by multivalue value expression are all also present in the values yielded by anothermultivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", + "description" : "Checks if the values yielded by multivalue value expression are all present in the values yielded by the other multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", "signatures" : [ { "params" : [ diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md b/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md index 4d22c719516d4..190bb4231bcbc 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md @@ -1,7 +1,7 @@ % This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. ### MV CONTAINS ALL -Checks if the values yielded by multivalue value expression are all also present in the values yielded by anothermultivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. +Checks if the values yielded by multivalue value expression are all present in the values yielded by the other multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. ```esql ROW a = "a", b = ["a", "b", "c"] From 1a00c73c726880aec929bba91127a648e79c6b58 Mon Sep 17 00:00:00 2001 From: Michael Bischoff Date: Tue, 19 Aug 2025 08:34:14 +0200 Subject: [PATCH 08/41] Update docs/changelog/133099.yaml --- docs/changelog/133099.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/133099.yaml diff --git a/docs/changelog/133099.yaml b/docs/changelog/133099.yaml new file mode 100644 index 0000000000000..f9c51e6fbabce --- /dev/null +++ b/docs/changelog/133099.yaml @@ -0,0 +1,5 @@ +pr: 133099 +summary: Add MV_CONTAINS_ALL ES:QL function +area: ES|QL +type: enhancement +issues: [] From d6f5e713cc0749bdb5ba9d9cd0d18712f0c23235 Mon Sep 17 00:00:00 2001 From: Michael Bischoff Date: Tue, 19 Aug 2025 08:35:50 +0200 Subject: [PATCH 09/41] Update docs/changelog/133099.yaml --- docs/changelog/133099.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/133099.yaml b/docs/changelog/133099.yaml index f9c51e6fbabce..0592a854ce24a 100644 --- a/docs/changelog/133099.yaml +++ b/docs/changelog/133099.yaml @@ -1,5 +1,5 @@ pr: 133099 summary: Add MV_CONTAINS_ALL ES:QL function area: ES|QL -type: enhancement +type: "enhancement, feature" issues: [] From b4c31b9600df226d0d5e9ad7d8e3f4ede70b6dfd Mon Sep 17 00:00:00 2001 From: Michael Bischoff Date: Tue, 19 Aug 2025 13:39:20 +0200 Subject: [PATCH 10/41] Update docs/changelog/133099.yaml --- docs/changelog/133099.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/133099.yaml b/docs/changelog/133099.yaml index 0592a854ce24a..f9c51e6fbabce 100644 --- a/docs/changelog/133099.yaml +++ b/docs/changelog/133099.yaml @@ -1,5 +1,5 @@ pr: 133099 summary: Add MV_CONTAINS_ALL ES:QL function area: ES|QL -type: "enhancement, feature" +type: enhancement issues: [] From 1a1b88d411bf576b318be6494df7df78972f1a10 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Tue, 19 Aug 2025 18:51:20 +0200 Subject: [PATCH 11/41] Readding skipped markers as else the test gets run in the backwards compatibility test environment. - not sure how to test it as, I feel like the version should be main on main / dev. Doing the dance for now. --- .../qa/testFixtures/src/main/resources/string.csv-spec | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index c4562fa62dc25..6e85f1c3cbdcd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1931,7 +1931,7 @@ l1:integer | l2:integer null | 0 ; -mvContainsAll#[skip:-9.2.99,reason:new base64 function added in 9.3] +mvContainsAll#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] // tag::mv_contains_all[] ROW a = "a", b = ["a", "b", "c"] | EVAL a_subset_of_b = mv_contains_all(b, a) @@ -1944,7 +1944,7 @@ a | [a, b, c] | true // end::mv_contains_all-result[] ; -mvContainsAll_bothsides#[skip:-9.2.99,reason:new base64 function added in 9.3] +mvContainsAll_bothsides#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] // tag::mv_contains_all_bothsides[] ROW a = ["a","c"], b = ["a", "b", "c"] | EVAL a_subset_of_b = mv_contains_all(b, a) @@ -1957,7 +1957,7 @@ a:keyword | b:keyword | a_subset_of_b:boolean // end::mv_contains_all_bothsides-result[] ; -mvContainsAllCombinations#[skip:-9.2.99,reason:new base64 function added in 9.3] +mvContainsAllCombinations#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] ROW a = "a", b = ["a", "b", "c"], n = null | EVAL aa = mv_contains_all(a, a), @@ -1969,11 +1969,11 @@ ROW a = "a", b = ["a", "b", "c"], n = null nn = mv_contains_all(n,n) ; -a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:boolean +a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:null a | [a, b, c] | null | true | true | false | true | null | null | null ; -mvContainsAll_where#[skip:-9.2.99,reason:new base64 function added in 9.3] +mvContainsAll_where#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] // tag::mv_contains_all_where[] FROM airports | WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9 From 23544d4bba520d40de057836c8ff8dbe25fd7c93 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Wed, 20 Aug 2025 15:36:25 +0200 Subject: [PATCH 12/41] Update `mv_contains_all` docs to reflect the correct version and add lifecycle metadata Documentation rewording. Co-authored-by: Liam Thompson --- docs/changelog/133099.yaml | 2 +- .../testFixtures/src/main/resources/string.csv-spec | 8 ++++---- .../function/scalar/multivalue/MvContainsAll.java | 11 ++++++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/changelog/133099.yaml b/docs/changelog/133099.yaml index f9c51e6fbabce..3f158995b8ad1 100644 --- a/docs/changelog/133099.yaml +++ b/docs/changelog/133099.yaml @@ -1,5 +1,5 @@ pr: 133099 -summary: Add MV_CONTAINS_ALL ES:QL function +summary: Add MV_CONTAINS_ALL function area: ES|QL type: enhancement issues: [] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 6e85f1c3cbdcd..2e0fcc5fcf606 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1931,7 +1931,7 @@ l1:integer | l2:integer null | 0 ; -mvContainsAll#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] +mvContainsAll#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] // tag::mv_contains_all[] ROW a = "a", b = ["a", "b", "c"] | EVAL a_subset_of_b = mv_contains_all(b, a) @@ -1944,7 +1944,7 @@ a | [a, b, c] | true // end::mv_contains_all-result[] ; -mvContainsAll_bothsides#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] +mvContainsAll_bothsides#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] // tag::mv_contains_all_bothsides[] ROW a = ["a","c"], b = ["a", "b", "c"] | EVAL a_subset_of_b = mv_contains_all(b, a) @@ -1957,7 +1957,7 @@ a:keyword | b:keyword | a_subset_of_b:boolean // end::mv_contains_all_bothsides-result[] ; -mvContainsAllCombinations#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] +mvContainsAllCombinations#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] ROW a = "a", b = ["a", "b", "c"], n = null | EVAL aa = mv_contains_all(a, a), @@ -1973,7 +1973,7 @@ a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boo a | [a, b, c] | null | true | true | false | true | null | null | null ; -mvContainsAll_where#[skip:-9.2.99,reason:new mvContainsall function added in 9.3] +mvContainsAll_where#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] // tag::mv_contains_all_where[] FROM airports | WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index cdf7065aea861..a88963db954df 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -28,6 +28,8 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; @@ -52,12 +54,15 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp @FunctionInfo( returnType = "boolean", - description = "Checks if the values yielded by multivalue value expression are all present in the values yielded by the other " - + "multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", + description = "\"Checks if all values yielded by the second multivalue expression are present in the values yielded by " + + "the first multivalue expression. Returns a boolean, or null if either expression is null.", examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), - @Example(file = "string", tag = "mv_contains_all_where"), } + @Example(file = "string", tag = "mv_contains_all_where"), }, + appliesTo = { + @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") + } ) public MvContainsAll( Source source, From a1e1ba313000116a5e29696815bd6a1116ee460e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 20 Aug 2025 13:42:29 +0000 Subject: [PATCH 13/41] [CI] Auto commit changes from spotless --- .../function/scalar/multivalue/MvContainsAll.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index a88963db954df..2ebb5e2273661 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -54,15 +54,13 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp @FunctionInfo( returnType = "boolean", - description = "\"Checks if all values yielded by the second multivalue expression are present in the values yielded by " + - "the first multivalue expression. Returns a boolean, or null if either expression is null.", + description = "\"Checks if all values yielded by the second multivalue expression are present in the values yielded by " + + "the first multivalue expression. Returns a boolean, or null if either expression is null.", examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), @Example(file = "string", tag = "mv_contains_all_where"), }, - appliesTo = { - @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") - } + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") } ) public MvContainsAll( Source source, From cd7625e91485219236c2d756c5c4bfba66902ba3 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Sat, 23 Aug 2025 03:46:13 +0200 Subject: [PATCH 14/41] Processing review, changing null handling --- .../src/main/resources/string.csv-spec | 33 +++++---- .../xpack/esql/action/EsqlCapabilities.java | 7 ++ .../scalar/multivalue/MvContainsAll.java | 70 +++++++++++++------ 3 files changed, 74 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 2e0fcc5fcf606..cb8cd987322ee 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1931,33 +1931,37 @@ l1:integer | l2:integer null | 0 ; -mvContainsAll#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] +mvContainsAll +required_capability: fn_mv_contains_all // tag::mv_contains_all[] -ROW a = "a", b = ["a", "b", "c"] -| EVAL a_subset_of_b = mv_contains_all(b, a) +ROW set = ["a", "b", "c"], element = "a" +| EVAL set_contains_element = mv_contains_all(set, element) // end::mv_contains_all[] ; // tag::mv_contains_all-result[] -a:keyword | b:keyword | a_subset_of_b:boolean -a | [a, b, c] | true +set:keyword | element:keyword | set_contains_element:boolean +[a, b, c] | a | true // end::mv_contains_all-result[] ; -mvContainsAll_bothsides#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] +mvContainsAll_bothsides +required_capability: fn_mv_contains_all // tag::mv_contains_all_bothsides[] -ROW a = ["a","c"], b = ["a", "b", "c"] -| EVAL a_subset_of_b = mv_contains_all(b, a) +ROW setA = ["a","c"], setB = ["a", "b", "c"] +| EVAL a_subset_of_b = mv_contains_all(setB, setA) +| EVAL b_subset_of_a = mv_contains_all(setA, setB) // end::mv_contains_all_bothsides[] ; // tag::mv_contains_all_bothsides-result[] -a:keyword | b:keyword | a_subset_of_b:boolean -[a, c] | [a, b, c] | true +setA:keyword | setB:keyword | a_subset_of_b:boolean | b_subset_of_a:boolean +[a, c] | [a, b, c] | true | false // end::mv_contains_all_bothsides-result[] ; -mvContainsAllCombinations#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] +mvContainsAllCombinations +required_capability: fn_mv_contains_all ROW a = "a", b = ["a", "b", "c"], n = null | EVAL aa = mv_contains_all(a, a), @@ -1969,11 +1973,12 @@ ROW a = "a", b = ["a", "b", "c"], n = null nn = mv_contains_all(n,n) ; -a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:null -a | [a, b, c] | null | true | true | false | true | null | null | null +a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:boolean +a | [a, b, c] | null | true | true | false | true | false | true | true ; -mvContainsAll_where#[skip:-9.1.99,reason:new mvContainsall function added in 9.2] +mvContainsAll_where +required_capability: fn_mv_contains_all // tag::mv_contains_all_where[] FROM airports | WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index cdef9f8c33cbd..1961bdd77bb43 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -245,6 +245,13 @@ public enum Cap { */ FN_MONTH_NAME, + + /** + * support for MV_CONTAINS_ALL function + */ + FN_MV_CONTAINS_ALL, + + /** * Fixes for multiple functions not serializing their source, and emitting warnings with wrong line number and text. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index 2ebb5e2273661..af44da982ec9b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -15,13 +15,17 @@ import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -50,17 +54,16 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp "MvContainsAll", MvContainsAll::new ); - private DataType dataType; @FunctionInfo( returnType = "boolean", - description = "\"Checks if all values yielded by the second multivalue expression are present in the values yielded by " - + "the first multivalue expression. Returns a boolean, or null if either expression is null.", + description = "Checks if all values yielded by the second multivalue expression are present in the values yielded by " + + "the first multivalue expression. Returns a boolean. Null values are treated as an empty set.", examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), - @Example(file = "string", tag = "mv_contains_all_where"), }, - appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") } + @Example(file = "string", tag = "mv_contains_all_where"),}, + appliesTo = {@FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0")} ) public MvContainsAll( Source source, @@ -81,7 +84,7 @@ public MvContainsAll( "long", "text", "unsigned_long", - "version" }, + "version"}, description = "Multivalue expression." ) Expression superset, @Param( @@ -101,7 +104,7 @@ public MvContainsAll( "long", "text", "unsigned_long", - "version" }, + "version"}, description = "Multivalue expression." ) Expression subset ) { @@ -127,9 +130,7 @@ protected TypeResolution resolveType() { if (resolution.unresolved()) { return resolution; } - dataType = left().dataType() == DataType.NULL ? DataType.NULL : DataType.BOOLEAN; if (left().dataType() == DataType.NULL) { - dataType = right().dataType() == DataType.NULL ? DataType.NULL : DataType.BOOLEAN; return isRepresentableExceptCounters(right(), sourceText(), SECOND); } return isType(right(), t -> t.noText() == left().dataType().noText(), sourceText(), SECOND, left().dataType().noText().typeName()); @@ -137,14 +138,16 @@ protected TypeResolution resolveType() { @Override public DataType dataType() { - if (dataType == null) { - resolveType(); - } - return dataType; + return DataType.BOOLEAN; + } + + @Override + public Nullability nullable() { + return Nullability.FALSE; } @Override - protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) { + protected MvContainsAll replaceChildren(Expression newLeft, Expression newRight) { return new MvContainsAll(source(), newLeft, newRight); } @@ -162,7 +165,7 @@ public Object fold(FoldContext ctx) { public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var supersetType = PlannerUtils.toElementType(left().dataType()); var subsetType = PlannerUtils.toElementType(right().dataType()); - if (supersetType != subsetType) { + if (supersetType != ElementType.NULL && subsetType != ElementType.NULL && supersetType != subsetType) { throw new EsqlIllegalArgumentException( "Incompatible data types for MvContainsAll, superset type({}) value({}) and subset type({}) value({}) don't match.", supersetType, @@ -177,11 +180,12 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { case DOUBLE -> new MvContainsAllDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case INT -> new MvContainsAllIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case LONG -> new MvContainsAllLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; + case NULL -> new MvContainsAllNullEvaluator(toEvaluator.apply(right())); default -> throw EsqlIllegalArgumentException.illegalDataType(dataType()); }; } + @Evaluator(extraName = "Int") static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt)); @@ -228,10 +232,10 @@ static void appendTo(BooleanBlock.Builder builder, Boolean bool) { * A block is considered a subset if the superset contains values that test equal for all the values in the subset, independent of * order. Duplicates are ignored in the sense that for each duplicate in the subset, we will search/match against the first/any value * in the superset. + * * @param superset block to check against - * @param subset block containing values that should be present in the other block. - * @return {@code true} if the given blocks are a superset and subset to each other, {@code false} if not and {@code null} if the subset - * or superset contains only null values. + * @param subset block containing values that should be present in the other block. + * @return {@code true} if the given blocks are a superset and subset to each other, {@code false} if not. */ static Boolean containsAll( BlockType superset, @@ -242,8 +246,8 @@ static Boolean containsAll( if (superset == subset) { return true; } - if (subset.areAllValuesNull() || superset.areAllValuesNull()) { - return null; + if (subset.areAllValuesNull()) { + return true; } final var subsetCount = subset.getValueCount(position); @@ -283,4 +287,26 @@ static boolean hasValue( interface ValueExtractor { Type extractValue(BlockType block, int position); } + + private record MvContainsAllNullEvaluator(ExpressionEvaluator.Factory toEvaluator) implements ExpressionEvaluator.Factory { + @Override + public ExpressionEvaluator get(DriverContext context) { + return new ExpressionEvaluator() { + final ExpressionEvaluator subsetField = toEvaluator.get(context); + + @Override + public Block eval(Page page) { + try (Block block = subsetField.eval(page)) { + var position = page.getPositionCount(); + return context.blockFactory().newConstantBooleanBlockWith(block.isNull(position), position); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(subsetField); + } + }; + } + } } From 979aac9abc86c526a3b802ade6aa200b5f7dd3ee Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sat, 23 Aug 2025 01:54:13 +0000 Subject: [PATCH 15/41] [CI] Auto commit changes from spotless --- .../xpack/esql/action/EsqlCapabilities.java | 2 -- .../function/scalar/multivalue/MvContainsAll.java | 9 ++++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 1961bdd77bb43..5e6acf2aa8c42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -245,13 +245,11 @@ public enum Cap { */ FN_MONTH_NAME, - /** * support for MV_CONTAINS_ALL function */ FN_MV_CONTAINS_ALL, - /** * Fixes for multiple functions not serializing their source, and emitting warnings with wrong line number and text. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index af44da982ec9b..c9c62730f3710 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -62,8 +62,8 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp examples = { @Example(file = "string", tag = "mv_contains_all"), @Example(file = "string", tag = "mv_contains_all_bothsides"), - @Example(file = "string", tag = "mv_contains_all_where"),}, - appliesTo = {@FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0")} + @Example(file = "string", tag = "mv_contains_all_where"), }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") } ) public MvContainsAll( Source source, @@ -84,7 +84,7 @@ public MvContainsAll( "long", "text", "unsigned_long", - "version"}, + "version" }, description = "Multivalue expression." ) Expression superset, @Param( @@ -104,7 +104,7 @@ public MvContainsAll( "long", "text", "unsigned_long", - "version"}, + "version" }, description = "Multivalue expression." ) Expression subset ) { @@ -185,7 +185,6 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { }; } - @Evaluator(extraName = "Int") static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt)); From 254fc59340878b4a889314ac476e32e8c0d3ec7a Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Sat, 23 Aug 2025 20:16:42 +0200 Subject: [PATCH 16/41] Slightly more efficient null handling and fix null test logic --- .../scalar/multivalue/MvContainsAll.java | 19 ++++- .../scalar/multivalue/MvContainsAllTests.java | 81 ++++++++++++++++++- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index c9c62730f3710..de494e0cde99f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -174,13 +174,15 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { right() ); } + if(supersetType == ElementType.NULL || subsetType == ElementType.NULL) { + return new MvContainsAllNullEvaluator(toEvaluator.apply(right())); + } return switch (supersetType) { case BOOLEAN -> new MvContainsAllBooleanEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case BYTES_REF -> new MvContainsAllBytesRefEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case DOUBLE -> new MvContainsAllDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case INT -> new MvContainsAllIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case LONG -> new MvContainsAllLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - case NULL -> new MvContainsAllNullEvaluator(toEvaluator.apply(right())); default -> throw EsqlIllegalArgumentException.illegalDataType(dataType()); }; } @@ -287,11 +289,12 @@ interface ValueExtractor { Type extractValue(BlockType block, int position); } - private record MvContainsAllNullEvaluator(ExpressionEvaluator.Factory toEvaluator) implements ExpressionEvaluator.Factory { + private record MvContainsAllNullEvaluator(ExpressionEvaluator.Factory subsetFieldEvaluator) implements ExpressionEvaluator.Factory { + @Override public ExpressionEvaluator get(DriverContext context) { return new ExpressionEvaluator() { - final ExpressionEvaluator subsetField = toEvaluator.get(context); + final ExpressionEvaluator subsetField = subsetFieldEvaluator.get(context); @Override public Block eval(Page page) { @@ -305,7 +308,17 @@ public Block eval(Page page) { public void close() { Releasables.closeExpectNoException(subsetField); } + + @Override + public String toString() { + return "MvContainsAllNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + } }; } + + @Override + public String toString() { + return "MvContainsAllNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java index 59a6f7dc08290..f72979ccedf90 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java @@ -19,14 +19,20 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TypedData.MULTI_ROW_NULL; +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TypedData.NULL; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; public class MvContainsAllTests extends AbstractScalarFunctionTestCase { @@ -42,7 +48,14 @@ public static Iterable parameters() { longs(suppliers); doubles(suppliers); bytesRefs(suppliers); - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); + + return parameterSuppliersFromTypedData(anyNullIsNull( + suppliers, + (nullPosition, nullValueDataType, original) -> false + && nullValueDataType == DataType.NULL + && original.getData().size() == 1 ? DataType.NULL : original.expectedType(), + (nullPosition, nullData, original) -> original + )); } @Override @@ -270,4 +283,70 @@ private static void bytesRefs(List suppliers) { })); } + // Adjusted from static method anyNullIsNull in {@code AbstractScalarFunctionTestCase#} + protected static List anyNullIsNull( + List testCaseSuppliers, + ExpectedType expectedType, + ExpectedEvaluatorToString evaluatorToString + ) { + List suppliers = new ArrayList<>(testCaseSuppliers.size()); + suppliers.addAll(testCaseSuppliers); + + /* + * For each original test case, add as many copies as there were + * arguments, replacing one of the arguments with null and keeping + * the others. + * + * Also, if this was the first time we saw the signature we copy it + * *again*, replacing the argument with null, but annotating the + * argument’s type as `null` explicitly. + */ + Set> uniqueSignatures = new HashSet<>(); + for (TestCaseSupplier original : testCaseSuppliers) { + boolean firstTimeSeenSignature = uniqueSignatures.add(original.types()); + for (int typeIndex = 0; typeIndex < original.types().size(); typeIndex++) { + int nullPosition = typeIndex; + + suppliers.add(new TestCaseSupplier(original.name() + " null in " + nullPosition, original.types(), () -> { + TestCaseSupplier.TestCase originalTestCase = original.get(); + List data = new ArrayList<>(originalTestCase.getData()); + data.set(nullPosition, NULL); + TestCaseSupplier.TypedData nulledData = originalTestCase.getData().get(nullPosition); + return new TestCaseSupplier.TestCase( + data, + evaluatorToString.evaluatorToString(nullPosition, nulledData, originalTestCase.evaluatorToString()), + expectedType.expectedType(nullPosition, nulledData.type(), originalTestCase), + equalTo(nullPosition == 1) + ); + })); + + if (firstTimeSeenSignature) { + var typesWithNull = new ArrayList<>(original.types()); + typesWithNull.set(nullPosition, DataType.NULL); + boolean newSignature = uniqueSignatures.add(typesWithNull); + if (newSignature) { + suppliers.add(new TestCaseSupplier(typesWithNull, () -> { + TestCaseSupplier.TestCase originalTestCase = original.get(); + var typeDataWithNull = new ArrayList<>(originalTestCase.getData()); + typeDataWithNull.set(nullPosition, typeDataWithNull.get(nullPosition).isMultiRow() ? MULTI_ROW_NULL : NULL); + return new TestCaseSupplier.TestCase( + typeDataWithNull, + "MvContainsAllNullEvaluator[subsetField=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(nullPosition == 1) + ); + })); + } + } + } + } + + return suppliers; + } + + // We always return a boolean. + @Override + protected Matcher allNullsMatcher() { + return anyOf(equalTo(false),equalTo(true)); + } } From b0e942a32cfb2dc090dc1842e82344569326e4e3 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sat, 23 Aug 2025 18:26:35 +0000 Subject: [PATCH 17/41] [CI] Auto commit changes from spotless --- .../scalar/multivalue/MvContainsAll.java | 2 +- .../scalar/multivalue/MvContainsAllTests.java | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java index de494e0cde99f..8a35d266ea1d7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java @@ -174,7 +174,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { right() ); } - if(supersetType == ElementType.NULL || subsetType == ElementType.NULL) { + if (supersetType == ElementType.NULL || subsetType == ElementType.NULL) { return new MvContainsAllNullEvaluator(toEvaluator.apply(right())); } return switch (supersetType) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java index f72979ccedf90..dc85e313b98cd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java @@ -49,13 +49,15 @@ public static Iterable parameters() { doubles(suppliers); bytesRefs(suppliers); - return parameterSuppliersFromTypedData(anyNullIsNull( - suppliers, - (nullPosition, nullValueDataType, original) -> false - && nullValueDataType == DataType.NULL - && original.getData().size() == 1 ? DataType.NULL : original.expectedType(), - (nullPosition, nullData, original) -> original - )); + return parameterSuppliersFromTypedData( + anyNullIsNull( + suppliers, + (nullPosition, nullValueDataType, original) -> false && nullValueDataType == DataType.NULL && original.getData().size() == 1 + ? DataType.NULL + : original.expectedType(), + (nullPosition, nullData, original) -> original + ) + ); } @Override @@ -347,6 +349,6 @@ protected static List anyNullIsNull( // We always return a boolean. @Override protected Matcher allNullsMatcher() { - return anyOf(equalTo(false),equalTo(true)); + return anyOf(equalTo(false), equalTo(true)); } } From b0c959db205f00e35ffb00cb54911c3845fffdbd Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Sun, 24 Aug 2025 14:01:51 +0200 Subject: [PATCH 18/41] - Renamed MV_CONTAINS_ALL to MV_CONTAINS - Fixing tests by removing logic to return null if all parameters are null. The standard generator had to be circumvented, should follow up with separate PR to make it more intelligent to avoid it. - Overwritten part of the test methods to avoid the null expectation. --- docs/changelog/133099.yaml | 2 +- .../functions/description/mv_contains.md | 6 + .../functions/description/mv_contains_all.md | 6 - .../functions/examples/mv_contains.md | 34 + .../functions/examples/mv_contains_all.md | 33 - .../_snippets/functions/layout/mv_contains.md | 26 + .../functions/layout/mv_contains_all.md | 23 - .../{mv_contains_all.md => mv_contains.md} | 0 .../{mv_contains_all.md => mv_contains.md} | 0 .../esql/images/functions/mv_contains.svg | 1 + .../esql/images/functions/mv_contains_all.svg | 1 - ...{mv_contains_all.json => mv_contains.json} | 10 +- .../esql/kibana/docs/functions/mv_contains.md | 9 + .../kibana/docs/functions/mv_contains_all.md | 9 - .../src/main/resources/string.csv-spec | 62 +- .../MvContainsAllBooleanEvaluator.java | 116 --- .../MvContainsAllBytesRefEvaluator.java | 118 --- .../MvContainsAllDoubleEvaluator.java | 117 --- .../multivalue/MvContainsAllIntEvaluator.java | 117 --- .../MvContainsAllLongEvaluator.java | 117 --- .../xpack/esql/action/EsqlCapabilities.java | 5 +- .../function/EsqlFunctionRegistry.java | 4 +- .../scalar/multivalue/MvContains.java | 704 ++++++++++++++++++ .../scalar/multivalue/MvContainsAll.java | 324 -------- .../multivalue/MvFunctionWritables.java | 2 +- ...orTests.java => MvContainsErrorTests.java} | 6 +- ...java => MvContainsSerializationTests.java} | 10 +- ...ainsAllTests.java => MvContainsTests.java} | 97 +-- 28 files changed, 883 insertions(+), 1076 deletions(-) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/mv_contains.md delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains.md delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains.md delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md rename docs/reference/query-languages/esql/_snippets/functions/parameters/{mv_contains_all.md => mv_contains.md} (100%) rename docs/reference/query-languages/esql/_snippets/functions/types/{mv_contains_all.md => mv_contains.md} (100%) create mode 100644 docs/reference/query-languages/esql/images/functions/mv_contains.svg delete mode 100644 docs/reference/query-languages/esql/images/functions/mv_contains_all.svg rename docs/reference/query-languages/esql/kibana/definition/functions/{mv_contains_all.json => mv_contains.json} (92%) create mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/mv_contains.md delete mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md delete mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java delete mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java delete mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java delete mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java delete mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/{MvContainsAllErrorTests.java => MvContainsErrorTests.java} (86%) rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/{MvContainsAllSerializationTests.java => MvContainsSerializationTests.java} (75%) rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/{MvContainsAllTests.java => MvContainsTests.java} (79%) diff --git a/docs/changelog/133099.yaml b/docs/changelog/133099.yaml index 3f158995b8ad1..8e81128073787 100644 --- a/docs/changelog/133099.yaml +++ b/docs/changelog/133099.yaml @@ -1,5 +1,5 @@ pr: 133099 -summary: Add MV_CONTAINS_ALL function +summary: Add MV_CONTAINS function area: ES|QL type: enhancement issues: [] diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains.md b/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains.md new file mode 100644 index 0000000000000..da02211ed38db --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Checks if all values yielded by the second multivalue expression are present in the values yielded by the first multivalue expression. Returns a boolean. Null values are treated as an empty set. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md deleted file mode 100644 index f08a0913b77e5..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/description/mv_contains_all.md +++ /dev/null @@ -1,6 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Description** - -Checks if the values yielded by multivalue value expression are all present in the values yielded by the other multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. - diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains.md b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains.md new file mode 100644 index 0000000000000..df77ca463c9be --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains.md @@ -0,0 +1,34 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Examples** + +```esql +ROW set = ["a", "b", "c"], element = "a" +| EVAL set_contains_element = mv_contains(set, element) +``` + +| set:keyword | element:keyword | set_contains_element:boolean | +| --- | --- | --- | +| [a, b, c] | a | true | + +```esql +ROW setA = ["a","c"], setB = ["a", "b", "c"] +| EVAL a_subset_of_b = mv_contains(setB, setA) +| EVAL b_subset_of_a = mv_contains(setA, setB) +``` + +| setA:keyword | setB:keyword | a_subset_of_b:boolean | b_subset_of_a:boolean | +| --- | --- | --- | --- | +| [a, c] | [a, b, c] | true | false | + +```esql +FROM airports +| WHERE mv_contains(type, ["major","military"]) AND scalerank == 9 +| KEEP scalerank, name, country +``` + +| scalerank:integer | name:text | country:keyword | +| --- | --- | --- | +| 9 | Chandigarh Int'l | India | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md deleted file mode 100644 index aaf2ec00ee7cb..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/mv_contains_all.md +++ /dev/null @@ -1,33 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Examples** - -```esql -ROW a = "a", b = ["a", "b", "c"] -| EVAL a_subset_of_b = mv_contains_all(b, a) -``` - -| a:keyword | b:keyword | a_subset_of_b:boolean | -| --- | --- | --- | -| a | [a, b, c] | true | - -```esql -ROW a = ["a","c"], b = ["a", "b", "c"] -| EVAL a_subset_of_b = mv_contains_all(b, a) -``` - -| a:keyword | b:keyword | a_subset_of_b:boolean | -| --- | --- | --- | -| [a, c] | [a, b, c] | true | - -```esql -FROM airports -| WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9 -| KEEP scalerank, name, country -``` - -| scalerank:integer | name:text | country:keyword | -| --- | --- | --- | -| 9 | Chandigarh Int'l | India | - - diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains.md b/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains.md new file mode 100644 index 0000000000000..aee13481b435f --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains.md @@ -0,0 +1,26 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `MV_CONTAINS` [esql-mv_contains] +```{applies_to} +stack: preview 9.2.0 +``` + +**Syntax** + +:::{image} ../../../images/functions/mv_contains.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/mv_contains.md +::: + +:::{include} ../description/mv_contains.md +::: + +:::{include} ../types/mv_contains.md +::: + +:::{include} ../examples/mv_contains.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md deleted file mode 100644 index 42bc9c85f0ebd..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/mv_contains_all.md +++ /dev/null @@ -1,23 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -## `MV_CONTAINS_ALL` [esql-mv_contains_all] - -**Syntax** - -:::{image} ../../../images/functions/mv_contains_all.svg -:alt: Embedded -:class: text-center -::: - - -:::{include} ../parameters/mv_contains_all.md -::: - -:::{include} ../description/mv_contains_all.md -::: - -:::{include} ../types/mv_contains_all.md -::: - -:::{include} ../examples/mv_contains_all.md -::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains.md similarity index 100% rename from docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains_all.md rename to docs/reference/query-languages/esql/_snippets/functions/parameters/mv_contains.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/mv_contains_all.md b/docs/reference/query-languages/esql/_snippets/functions/types/mv_contains.md similarity index 100% rename from docs/reference/query-languages/esql/_snippets/functions/types/mv_contains_all.md rename to docs/reference/query-languages/esql/_snippets/functions/types/mv_contains.md diff --git a/docs/reference/query-languages/esql/images/functions/mv_contains.svg b/docs/reference/query-languages/esql/images/functions/mv_contains.svg new file mode 100644 index 0000000000000..3a588496e392b --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/mv_contains.svg @@ -0,0 +1 @@ +MV_CONTAINS(superset,subset) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/images/functions/mv_contains_all.svg b/docs/reference/query-languages/esql/images/functions/mv_contains_all.svg deleted file mode 100644 index f7e5b3807492b..0000000000000 --- a/docs/reference/query-languages/esql/images/functions/mv_contains_all.svg +++ /dev/null @@ -1 +0,0 @@ -MV_CONTAINS_ALL(superset,subset) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains.json similarity index 92% rename from docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json rename to docs/reference/query-languages/esql/kibana/definition/functions/mv_contains.json index aba4c900335ca..0116939ba4f59 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains_all.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/mv_contains.json @@ -1,8 +1,8 @@ { "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", "type" : "scalar", - "name" : "mv_contains_all", - "description" : "Checks if the values yielded by multivalue value expression are all present in the values yielded by the other multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null.", + "name" : "mv_contains", + "description" : "Checks if all values yielded by the second multivalue expression are present in the values yielded by the first multivalue expression. Returns a boolean. Null values are treated as an empty set.", "signatures" : [ { "params" : [ @@ -312,9 +312,9 @@ } ], "examples" : [ - "ROW a = \"a\", b = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains_all(b, a)", - "ROW a = [\"a\",\"c\"], b = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains_all(b, a)", - "FROM airports\n| WHERE mv_contains_all(type, [\"major\",\"military\"]) AND scalerank == 9\n| KEEP scalerank, name, country" + "ROW set = [\"a\", \"b\", \"c\"], element = \"a\"\n| EVAL set_contains_element = mv_contains(set, element)", + "ROW setA = [\"a\",\"c\"], setB = [\"a\", \"b\", \"c\"]\n| EVAL a_subset_of_b = mv_contains(setB, setA)\n| EVAL b_subset_of_a = mv_contains(setA, setB)", + "FROM airports\n| WHERE mv_contains(type, [\"major\",\"military\"]) AND scalerank == 9\n| KEEP scalerank, name, country" ], "preview" : false, "snapshot_only" : false diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains.md b/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains.md new file mode 100644 index 0000000000000..4bc82881dc292 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### MV CONTAINS +Checks if all values yielded by the second multivalue expression are present in the values yielded by the first multivalue expression. Returns a boolean. Null values are treated as an empty set. + +```esql +ROW set = ["a", "b", "c"], element = "a" +| EVAL set_contains_element = mv_contains(set, element) +``` diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md b/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md deleted file mode 100644 index 190bb4231bcbc..0000000000000 --- a/docs/reference/query-languages/esql/kibana/docs/functions/mv_contains_all.md +++ /dev/null @@ -1,9 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -### MV CONTAINS ALL -Checks if the values yielded by multivalue value expression are all present in the values yielded by the other multivalue expression. The result is a boolean representing the outcome or null if either of the expressions where null. - -```esql -ROW a = "a", b = ["a", "b", "c"] -| EVAL a_subset_of_b = mv_contains_all(b, a) -``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index cb8cd987322ee..6e7619b0ab282 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1931,65 +1931,65 @@ l1:integer | l2:integer null | 0 ; -mvContainsAll -required_capability: fn_mv_contains_all -// tag::mv_contains_all[] +mvContains +required_capability: fn_mv_contains +// tag::mv_contains[] ROW set = ["a", "b", "c"], element = "a" -| EVAL set_contains_element = mv_contains_all(set, element) -// end::mv_contains_all[] +| EVAL set_contains_element = mv_contains(set, element) +// end::mv_contains[] ; -// tag::mv_contains_all-result[] +// tag::mv_contains-result[] set:keyword | element:keyword | set_contains_element:boolean [a, b, c] | a | true -// end::mv_contains_all-result[] +// end::mv_contains-result[] ; -mvContainsAll_bothsides -required_capability: fn_mv_contains_all -// tag::mv_contains_all_bothsides[] +mvContains_bothsides +required_capability: fn_mv_contains +// tag::mv_contains_bothsides[] ROW setA = ["a","c"], setB = ["a", "b", "c"] -| EVAL a_subset_of_b = mv_contains_all(setB, setA) -| EVAL b_subset_of_a = mv_contains_all(setA, setB) -// end::mv_contains_all_bothsides[] +| EVAL a_subset_of_b = mv_contains(setB, setA) +| EVAL b_subset_of_a = mv_contains(setA, setB) +// end::mv_contains_bothsides[] ; -// tag::mv_contains_all_bothsides-result[] +// tag::mv_contains_bothsides-result[] setA:keyword | setB:keyword | a_subset_of_b:boolean | b_subset_of_a:boolean [a, c] | [a, b, c] | true | false -// end::mv_contains_all_bothsides-result[] +// end::mv_contains_bothsides-result[] ; -mvContainsAllCombinations -required_capability: fn_mv_contains_all +mvContainsCombinations +required_capability: fn_mv_contains ROW a = "a", b = ["a", "b", "c"], n = null -| EVAL aa = mv_contains_all(a, a), - bb = mv_contains_all(b, b), - ab = mv_contains_all(a, b), - ba = mv_contains_all(b,a), - na = mv_contains_all(n, a), - an = mv_contains_all(a, n), - nn = mv_contains_all(n,n) +| EVAL aa = mv_contains(a, a), + bb = mv_contains(b, b), + ab = mv_contains(a, b), + ba = mv_contains(b,a), + na = mv_contains(n, a), + an = mv_contains(a, n), + nn = mv_contains(n,n) ; a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:boolean a | [a, b, c] | null | true | true | false | true | false | true | true ; -mvContainsAll_where -required_capability: fn_mv_contains_all -// tag::mv_contains_all_where[] +mvContains_where +required_capability: fn_mv_contains +// tag::mv_contains_where[] FROM airports -| WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9 +| WHERE mv_contains(type, ["major","military"]) AND scalerank == 9 | KEEP scalerank, name, country -// end::mv_contains_all_where[] +// end::mv_contains_where[] ; -// tag::mv_contains_all_where-result[] +// tag::mv_contains_where-result[] scalerank:integer | name:text | country:keyword 9 | Chandigarh Int'l | India -// end::mv_contains_all_where-result[] +// end::mv_contains_where-result[] ; mvAppend diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java deleted file mode 100644 index 4eb973c496eaf..0000000000000 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBooleanEvaluator.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; - -import java.lang.Override; -import java.lang.String; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.Warnings; -import org.elasticsearch.core.Releasables; -import org.elasticsearch.xpack.esql.core.tree.Source; - -/** - * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. - * This class is generated. Edit {@code EvaluatorImplementer} instead. - */ -public final class MvContainsAllBooleanEvaluator implements EvalOperator.ExpressionEvaluator { - private final Source source; - - private final EvalOperator.ExpressionEvaluator field1; - - private final EvalOperator.ExpressionEvaluator field2; - - private final DriverContext driverContext; - - private Warnings warnings; - - public MvContainsAllBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - this.driverContext = driverContext; - } - - @Override - public Block eval(Page page) { - try (BooleanBlock field1Block = (BooleanBlock) field1.eval(page)) { - try (BooleanBlock field2Block = (BooleanBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); - } - } - } - - public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - position: for (int p = 0; p < positionCount; p++) { - boolean allBlocksAreNulls = true; - if (!field1Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (allBlocksAreNulls) { - result.appendNull(); - continue position; - } - MvContainsAll.process(result, p, field1Block, field2Block); - } - return result.build(); - } - } - - @Override - public String toString() { - return "MvContainsAllBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - - @Override - public void close() { - Releasables.closeExpectNoException(field1, field2); - } - - private Warnings warnings() { - if (warnings == null) { - this.warnings = Warnings.createWarnings( - driverContext.warningsMode(), - source.source().getLineNumber(), - source.source().getColumnNumber(), - source.text() - ); - } - return warnings; - } - - static class Factory implements EvalOperator.ExpressionEvaluator.Factory { - private final Source source; - - private final EvalOperator.ExpressionEvaluator.Factory field1; - - private final EvalOperator.ExpressionEvaluator.Factory field2; - - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - } - - @Override - public MvContainsAllBooleanEvaluator get(DriverContext context) { - return new MvContainsAllBooleanEvaluator(source, field1.get(context), field2.get(context), context); - } - - @Override - public String toString() { - return "MvContainsAllBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - } -} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java deleted file mode 100644 index 8d345c367fb39..0000000000000 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllBytesRefEvaluator.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; - -import java.lang.Override; -import java.lang.String; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.Warnings; -import org.elasticsearch.core.Releasables; -import org.elasticsearch.xpack.esql.core.tree.Source; - -/** - * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. - * This class is generated. Edit {@code EvaluatorImplementer} instead. - */ -public final class MvContainsAllBytesRefEvaluator implements EvalOperator.ExpressionEvaluator { - private final Source source; - - private final EvalOperator.ExpressionEvaluator field1; - - private final EvalOperator.ExpressionEvaluator field2; - - private final DriverContext driverContext; - - private Warnings warnings; - - public MvContainsAllBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - this.driverContext = driverContext; - } - - @Override - public Block eval(Page page) { - try (BytesRefBlock field1Block = (BytesRefBlock) field1.eval(page)) { - try (BytesRefBlock field2Block = (BytesRefBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); - } - } - } - - public BooleanBlock eval(int positionCount, BytesRefBlock field1Block, - BytesRefBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - position: for (int p = 0; p < positionCount; p++) { - boolean allBlocksAreNulls = true; - if (!field1Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (allBlocksAreNulls) { - result.appendNull(); - continue position; - } - MvContainsAll.process(result, p, field1Block, field2Block); - } - return result.build(); - } - } - - @Override - public String toString() { - return "MvContainsAllBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - - @Override - public void close() { - Releasables.closeExpectNoException(field1, field2); - } - - private Warnings warnings() { - if (warnings == null) { - this.warnings = Warnings.createWarnings( - driverContext.warningsMode(), - source.source().getLineNumber(), - source.source().getColumnNumber(), - source.text() - ); - } - return warnings; - } - - static class Factory implements EvalOperator.ExpressionEvaluator.Factory { - private final Source source; - - private final EvalOperator.ExpressionEvaluator.Factory field1; - - private final EvalOperator.ExpressionEvaluator.Factory field2; - - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - } - - @Override - public MvContainsAllBytesRefEvaluator get(DriverContext context) { - return new MvContainsAllBytesRefEvaluator(source, field1.get(context), field2.get(context), context); - } - - @Override - public String toString() { - return "MvContainsAllBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - } -} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java deleted file mode 100644 index 9cf90bdc3786b..0000000000000 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllDoubleEvaluator.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; - -import java.lang.Override; -import java.lang.String; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.DoubleBlock; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.Warnings; -import org.elasticsearch.core.Releasables; -import org.elasticsearch.xpack.esql.core.tree.Source; - -/** - * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. - * This class is generated. Edit {@code EvaluatorImplementer} instead. - */ -public final class MvContainsAllDoubleEvaluator implements EvalOperator.ExpressionEvaluator { - private final Source source; - - private final EvalOperator.ExpressionEvaluator field1; - - private final EvalOperator.ExpressionEvaluator field2; - - private final DriverContext driverContext; - - private Warnings warnings; - - public MvContainsAllDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - this.driverContext = driverContext; - } - - @Override - public Block eval(Page page) { - try (DoubleBlock field1Block = (DoubleBlock) field1.eval(page)) { - try (DoubleBlock field2Block = (DoubleBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); - } - } - } - - public BooleanBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - position: for (int p = 0; p < positionCount; p++) { - boolean allBlocksAreNulls = true; - if (!field1Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (allBlocksAreNulls) { - result.appendNull(); - continue position; - } - MvContainsAll.process(result, p, field1Block, field2Block); - } - return result.build(); - } - } - - @Override - public String toString() { - return "MvContainsAllDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - - @Override - public void close() { - Releasables.closeExpectNoException(field1, field2); - } - - private Warnings warnings() { - if (warnings == null) { - this.warnings = Warnings.createWarnings( - driverContext.warningsMode(), - source.source().getLineNumber(), - source.source().getColumnNumber(), - source.text() - ); - } - return warnings; - } - - static class Factory implements EvalOperator.ExpressionEvaluator.Factory { - private final Source source; - - private final EvalOperator.ExpressionEvaluator.Factory field1; - - private final EvalOperator.ExpressionEvaluator.Factory field2; - - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - } - - @Override - public MvContainsAllDoubleEvaluator get(DriverContext context) { - return new MvContainsAllDoubleEvaluator(source, field1.get(context), field2.get(context), context); - } - - @Override - public String toString() { - return "MvContainsAllDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - } -} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java deleted file mode 100644 index 0d5a41f992d91..0000000000000 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllIntEvaluator.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; - -import java.lang.Override; -import java.lang.String; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.IntBlock; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.Warnings; -import org.elasticsearch.core.Releasables; -import org.elasticsearch.xpack.esql.core.tree.Source; - -/** - * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. - * This class is generated. Edit {@code EvaluatorImplementer} instead. - */ -public final class MvContainsAllIntEvaluator implements EvalOperator.ExpressionEvaluator { - private final Source source; - - private final EvalOperator.ExpressionEvaluator field1; - - private final EvalOperator.ExpressionEvaluator field2; - - private final DriverContext driverContext; - - private Warnings warnings; - - public MvContainsAllIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - this.driverContext = driverContext; - } - - @Override - public Block eval(Page page) { - try (IntBlock field1Block = (IntBlock) field1.eval(page)) { - try (IntBlock field2Block = (IntBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); - } - } - } - - public BooleanBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - position: for (int p = 0; p < positionCount; p++) { - boolean allBlocksAreNulls = true; - if (!field1Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (allBlocksAreNulls) { - result.appendNull(); - continue position; - } - MvContainsAll.process(result, p, field1Block, field2Block); - } - return result.build(); - } - } - - @Override - public String toString() { - return "MvContainsAllIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - - @Override - public void close() { - Releasables.closeExpectNoException(field1, field2); - } - - private Warnings warnings() { - if (warnings == null) { - this.warnings = Warnings.createWarnings( - driverContext.warningsMode(), - source.source().getLineNumber(), - source.source().getColumnNumber(), - source.text() - ); - } - return warnings; - } - - static class Factory implements EvalOperator.ExpressionEvaluator.Factory { - private final Source source; - - private final EvalOperator.ExpressionEvaluator.Factory field1; - - private final EvalOperator.ExpressionEvaluator.Factory field2; - - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - } - - @Override - public MvContainsAllIntEvaluator get(DriverContext context) { - return new MvContainsAllIntEvaluator(source, field1.get(context), field2.get(context), context); - } - - @Override - public String toString() { - return "MvContainsAllIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - } -} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java deleted file mode 100644 index 2e7cb847f52ce..0000000000000 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllLongEvaluator.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; - -import java.lang.Override; -import java.lang.String; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.Warnings; -import org.elasticsearch.core.Releasables; -import org.elasticsearch.xpack.esql.core.tree.Source; - -/** - * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvContainsAll}. - * This class is generated. Edit {@code EvaluatorImplementer} instead. - */ -public final class MvContainsAllLongEvaluator implements EvalOperator.ExpressionEvaluator { - private final Source source; - - private final EvalOperator.ExpressionEvaluator field1; - - private final EvalOperator.ExpressionEvaluator field2; - - private final DriverContext driverContext; - - private Warnings warnings; - - public MvContainsAllLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - this.driverContext = driverContext; - } - - @Override - public Block eval(Page page) { - try (LongBlock field1Block = (LongBlock) field1.eval(page)) { - try (LongBlock field2Block = (LongBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); - } - } - } - - public BooleanBlock eval(int positionCount, LongBlock field1Block, LongBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - position: for (int p = 0; p < positionCount; p++) { - boolean allBlocksAreNulls = true; - if (!field1Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; - } - if (allBlocksAreNulls) { - result.appendNull(); - continue position; - } - MvContainsAll.process(result, p, field1Block, field2Block); - } - return result.build(); - } - } - - @Override - public String toString() { - return "MvContainsAllLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - - @Override - public void close() { - Releasables.closeExpectNoException(field1, field2); - } - - private Warnings warnings() { - if (warnings == null) { - this.warnings = Warnings.createWarnings( - driverContext.warningsMode(), - source.source().getLineNumber(), - source.source().getColumnNumber(), - source.text() - ); - } - return warnings; - } - - static class Factory implements EvalOperator.ExpressionEvaluator.Factory { - private final Source source; - - private final EvalOperator.ExpressionEvaluator.Factory field1; - - private final EvalOperator.ExpressionEvaluator.Factory field2; - - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { - this.source = source; - this.field1 = field1; - this.field2 = field2; - } - - @Override - public MvContainsAllLongEvaluator get(DriverContext context) { - return new MvContainsAllLongEvaluator(source, field1.get(context), field2.get(context), context); - } - - @Override - public String toString() { - return "MvContainsAllLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; - } - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 5e6acf2aa8c42..729522918f59d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -246,9 +246,10 @@ public enum Cap { FN_MONTH_NAME, /** - * support for MV_CONTAINS_ALL function + * support for MV_CONTAINS function + * Add MV_CONTAINS function #133099 */ - FN_MV_CONTAINS_ALL, + FN_MV_CONTAINS, /** * Fixes for multiple functions not serializing their source, and emitting warnings with wrong line number and text. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 24a6bf979cf30..c48208a5e69cd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -123,7 +123,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvContainsAll; +import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvContains; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvDedupe; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvFirst; @@ -451,7 +451,7 @@ private static FunctionDefinition[][] functions() { def(MvAppend.class, MvAppend::new, "mv_append"), def(MvAvg.class, MvAvg::new, "mv_avg"), def(MvConcat.class, MvConcat::new, "mv_concat"), - def(MvContainsAll.class, MvContainsAll::new, "mv_contains_all"), + def(MvContains.class, MvContains::new, "mv_contains"), def(MvCount.class, MvCount::new, "mv_count"), def(MvDedupe.class, MvDedupe::new, "mv_dedupe"), def(MvFirst.class, MvFirst::new, "mv_first"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java new file mode 100644 index 0000000000000..ea85b01249466 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -0,0 +1,704 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; + +import java.io.IOException; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; + +/** + * Reduce a multivalued field to a single valued field containing the count of values. + */ +public class MvContains extends BinaryScalarFunction implements EvaluatorMapper { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "MvContains", + MvContains::new + ); + + @FunctionInfo( + returnType = "boolean", + description = "Checks if all values yielded by the second multivalue expression are present in the values yielded by " + + "the first multivalue expression. Returns a boolean. Null values are treated as an empty set.", + examples = { + @Example(file = "string", tag = "mv_contains"), + @Example(file = "string", tag = "mv_contains_bothsides"), + @Example(file = "string", tag = "mv_contains_where"), }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") } + ) + public MvContains( + Source source, + @Param( + name = "superset", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "Multivalue expression." + ) Expression superset, + @Param( + name = "subset", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "Multivalue expression." + ) Expression subset + ) { + super(source, superset, subset); + } + + private MvContains(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isRepresentableExceptCounters(left(), sourceText(), FIRST); + if (resolution.unresolved()) { + return resolution; + } + if (left().dataType() == DataType.NULL) { + return isRepresentableExceptCounters(right(), sourceText(), SECOND); + } + return isType(right(), t -> t.noText() == left().dataType().noText(), sourceText(), SECOND, left().dataType().noText().typeName()); + } + + @Override + public DataType dataType() { + return DataType.BOOLEAN; + } + + @Override + public Nullability nullable() { + return Nullability.FALSE; + } + + @Override + protected MvContains replaceChildren(Expression newLeft, Expression newRight) { + return new MvContains(source(), newLeft, newRight); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, MvContains::new, left(), right()); + } + + @Override + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); + } + + @Override + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var supersetType = PlannerUtils.toElementType(left().dataType()); + var subsetType = PlannerUtils.toElementType(right().dataType()); + if (supersetType != ElementType.NULL && subsetType != ElementType.NULL && supersetType != subsetType) { + throw new EsqlIllegalArgumentException( + "Incompatible data types for MvContains, superset type({}) value({}) and subset type({}) value({}) don't match.", + supersetType, + left(), + subsetType, + right() + ); + } + if(supersetType == ElementType.NULL || subsetType == ElementType.NULL) { + return new MvContainsNullEvaluator(toEvaluator.apply(right())); + } + return switch (supersetType) { + case BOOLEAN -> new MvContainsBooleanEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case BYTES_REF -> new MvContainsBytesRefEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case DOUBLE -> new MvContainsDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case INT -> new MvContainsIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case LONG -> new MvContainsLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + default -> throw EsqlIllegalArgumentException.illegalDataType(dataType()); + }; + } + + // @Evaluator(extraName = "Int") see end of file. + static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { + appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt)); + } + + // @Evaluator(extraName = "Boolean") see end of file. + static void process(BooleanBlock.Builder builder, int position, BooleanBlock field1, BooleanBlock field2) { + appendTo(builder, containsAll(field1, field2, position, BooleanBlock::getBoolean)); + } + + // @Evaluator(extraName = "Long") see end of file. + static void process(BooleanBlock.Builder builder, int position, LongBlock field1, LongBlock field2) { + appendTo(builder, containsAll(field1, field2, position, LongBlock::getLong)); + } + + // @Evaluator(extraName = "Double") see end of file. + static void process(BooleanBlock.Builder builder, int position, DoubleBlock field1, DoubleBlock field2) { + appendTo(builder, containsAll(field1, field2, position, DoubleBlock::getDouble)); + } + + // @Evaluator(extraName = "BytesRef") see end of file. + static void process(BooleanBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) { + appendTo(builder, containsAll(field1, field2, position, (block, index) -> { + var ref = new BytesRef(); + // we pass in a reference, but sometimes we only get a return value, see ConstantBytesRefVector.getBytesRef + ref = block.getBytesRef(index, ref); + // pass empty ref as null + if (ref.length == 0) { + return null; + } + return ref; + })); + } + + static void appendTo(BooleanBlock.Builder builder, Boolean bool) { + if (bool == null) { + builder.appendNull(); + } else { + builder.beginPositionEntry().appendBoolean(bool).endPositionEntry(); + } + } + + /** + * A block is considered a subset if the superset contains values that test equal for all the values in the subset, independent of + * order. Duplicates are ignored in the sense that for each duplicate in the subset, we will search/match against the first/any value + * in the superset. + * + * @param superset block to check against + * @param subset block containing values that should be present in the other block. + * @return {@code true} if the given blocks are a superset and subset to each other, {@code false} if not. + */ + static Boolean containsAll( + BlockType superset, + BlockType subset, + final int position, + ValueExtractor valueExtractor + ) { + if (superset == subset) { + return true; + } + if (subset.areAllValuesNull()) { + return true; + } + + final var subsetCount = subset.getValueCount(position); + final var startIndex = subset.getFirstValueIndex(position); + for (int subsetIndex = startIndex; subsetIndex < startIndex + subsetCount; subsetIndex++) { + var value = valueExtractor.extractValue(subset, subsetIndex); + if (hasValue(superset, position, value, valueExtractor) == false) { + return false; + } + } + return true; + } + + /** + * Check if the block has the value at any of it's positions + * @param superset Block to search + * @param value to search for + * @return true if the supplied long value is in the supplied Block + */ + static boolean hasValue( + BlockType superset, + final int position, + Type value, + ValueExtractor valueExtractor + ) { + final var supersetCount = superset.getValueCount(position); + final var startIndex = superset.getFirstValueIndex(position); + for (int supersetIndex = startIndex; supersetIndex < startIndex + supersetCount; supersetIndex++) { + var element = valueExtractor.extractValue(superset, supersetIndex); + if (element != null && element.equals(value)) { + return true; + } + } + return false; + } + + interface ValueExtractor { + Type extractValue(BlockType block, int position); + } + + private static final class MvContainsNullEvaluator implements ExpressionEvaluator.Factory { + private final ExpressionEvaluator.Factory subsetFieldEvaluator; + + private MvContainsNullEvaluator(ExpressionEvaluator.Factory subsetFieldEvaluator) { + this.subsetFieldEvaluator = subsetFieldEvaluator; + } + + @Override + public ExpressionEvaluator get(DriverContext context) { + return new ExpressionEvaluator() { + final ExpressionEvaluator subsetField = subsetFieldEvaluator.get(context); + + @Override + public Block eval(Page page) { + try (Block block = subsetField.eval(page)) { + var position = page.getPositionCount(); + return context.blockFactory().newConstantBooleanBlockWith(block.isNull(position), position); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(subsetField); + } + + @Override + public String toString() { + return "MvContainsNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + } + }; + } + + @Override + public String toString() { + return "MvContainsNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + } + } + + /** + * Currently {@code EvaluatorImplementer} generates: + * if (allBlocksAreNulls) { + * result.appendNull(); + * continue position; + * } + * when all params are null, this violates our contract of always returning a boolean. + * It should probably also generate the warnings method conditionally - omitted here. + * TODO extend code generation to handle this case + */ + public static class MvContainsBooleanEvaluator implements EvalOperator.ExpressionEvaluator { + private final EvalOperator.ExpressionEvaluator field1; + private final EvalOperator.ExpressionEvaluator field2; + private final DriverContext driverContext; + + public MvContainsBooleanEvaluator(EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext) { + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BooleanBlock field1Block = (BooleanBlock) field1.eval(page)) { + try (BooleanBlock field2Block = (BooleanBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + MvContains.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory field1; + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsBooleanEvaluator get(DriverContext context) { + return new MvContainsBooleanEvaluator(field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } + } + + /** + * Currently {@code EvaluatorImplementer} generates: + * if (allBlocksAreNulls) { + * result.appendNull(); + * continue position; + * } + * when all params are null, this violates our contract of always returning a boolean. + * It should probably also generate the warnings method conditionally - omitted here. + * TODO extend code generation to handle this case + */ + public static class MvContainsBytesRefEvaluator implements EvalOperator.ExpressionEvaluator { + private final EvalOperator.ExpressionEvaluator field1; + private final EvalOperator.ExpressionEvaluator field2; + private final DriverContext driverContext; + + public MvContainsBytesRefEvaluator(EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext) { + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock field1Block = (BytesRefBlock) field1.eval(page)) { + try (BytesRefBlock field2Block = (BytesRefBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, BytesRefBlock field1Block, BytesRefBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + MvContains.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory field1; + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsBytesRefEvaluator get(DriverContext context) { + return new MvContainsBytesRefEvaluator(field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } + } + + /** + * Currently {@code EvaluatorImplementer} generates: + * if (allBlocksAreNulls) { + * result.appendNull(); + * continue position; + * } + * when all params are null, this violates our contract of always returning a boolean. + * It should probably also generate the warnings method conditionally - omitted here. + * TODO extend code generation to handle this case + */ + public static class MvContainsDoubleEvaluator implements EvalOperator.ExpressionEvaluator { + private final EvalOperator.ExpressionEvaluator field1; + private final EvalOperator.ExpressionEvaluator field2; + private final DriverContext driverContext; + + public MvContainsDoubleEvaluator(EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext) { + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (DoubleBlock field1Block = (DoubleBlock) field1.eval(page)) { + try (DoubleBlock field2Block = (DoubleBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + MvContains.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory field1; + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsDoubleEvaluator get(DriverContext context) { + return new MvContainsDoubleEvaluator(field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } + } + + /** + * Currently {@code EvaluatorImplementer} generates: + * if (allBlocksAreNulls) { + * result.appendNull(); + * continue position; + * } + * when all params are null, this violates our contract of always returning a boolean. + * It should probably also generate the warnings method conditionally - omitted here. + * TODO extend code generation to handle this case + */ + public static class MvContainsIntEvaluator implements EvalOperator.ExpressionEvaluator { + private final EvalOperator.ExpressionEvaluator field1; + private final EvalOperator.ExpressionEvaluator field2; + private final DriverContext driverContext; + + public MvContainsIntEvaluator(EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext) { + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (IntBlock field1Block = (IntBlock) field1.eval(page)) { + try (IntBlock field2Block = (IntBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + MvContains.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory field1; + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsIntEvaluator get(DriverContext context) { + return new MvContainsIntEvaluator(field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } + } + + /** + * Currently {@code EvaluatorImplementer} generates: + * if (allBlocksAreNulls) { + * result.appendNull(); + * continue position; + * } + * when all params are null, this violates our contract of always returning a boolean. + * It should probably also generate the warnings method conditionally - omitted here. + * TODO extend code generation to handle this case + */ + public static class MvContainsLongEvaluator implements EvalOperator.ExpressionEvaluator { + private final EvalOperator.ExpressionEvaluator field1; + private final EvalOperator.ExpressionEvaluator field2; + private final DriverContext driverContext; + + public MvContainsLongEvaluator(EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext) { + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock field1Block = (LongBlock) field1.eval(page)) { + try (LongBlock field2Block = (LongBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock field1Block, LongBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + MvContains.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvContainsLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory field1; + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvContainsLongEvaluator get(DriverContext context) { + return new MvContainsLongEvaluator(field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvContainsLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java deleted file mode 100644 index 8a35d266ea1d7..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.DoubleBlock; -import org.elasticsearch.compute.data.ElementType; -import org.elasticsearch.compute.data.IntBlock; -import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; -import org.elasticsearch.core.Releasables; -import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FoldContext; -import org.elasticsearch.xpack.esql.core.expression.Nullability; -import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; -import org.elasticsearch.xpack.esql.expression.function.Example; -import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; -import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; -import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; -import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.planner.PlannerUtils; - -import java.io.IOException; - -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; - -/** - * Reduce a multivalued field to a single valued field containing the count of values. - */ -public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapper { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( - Expression.class, - "MvContainsAll", - MvContainsAll::new - ); - - @FunctionInfo( - returnType = "boolean", - description = "Checks if all values yielded by the second multivalue expression are present in the values yielded by " - + "the first multivalue expression. Returns a boolean. Null values are treated as an empty set.", - examples = { - @Example(file = "string", tag = "mv_contains_all"), - @Example(file = "string", tag = "mv_contains_all_bothsides"), - @Example(file = "string", tag = "mv_contains_all_where"), }, - appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") } - ) - public MvContainsAll( - Source source, - @Param( - name = "superset", - type = { - "boolean", - "cartesian_point", - "cartesian_shape", - "date", - "date_nanos", - "double", - "geo_point", - "geo_shape", - "integer", - "ip", - "keyword", - "long", - "text", - "unsigned_long", - "version" }, - description = "Multivalue expression." - ) Expression superset, - @Param( - name = "subset", - type = { - "boolean", - "cartesian_point", - "cartesian_shape", - "date", - "date_nanos", - "double", - "geo_point", - "geo_shape", - "integer", - "ip", - "keyword", - "long", - "text", - "unsigned_long", - "version" }, - description = "Multivalue expression." - ) Expression subset - ) { - super(source, superset, subset); - } - - private MvContainsAll(StreamInput in) throws IOException { - super(in); - } - - @Override - public String getWriteableName() { - return ENTRY.name; - } - - @Override - protected TypeResolution resolveType() { - if (childrenResolved() == false) { - return new TypeResolution("Unresolved children"); - } - - TypeResolution resolution = isRepresentableExceptCounters(left(), sourceText(), FIRST); - if (resolution.unresolved()) { - return resolution; - } - if (left().dataType() == DataType.NULL) { - return isRepresentableExceptCounters(right(), sourceText(), SECOND); - } - return isType(right(), t -> t.noText() == left().dataType().noText(), sourceText(), SECOND, left().dataType().noText().typeName()); - } - - @Override - public DataType dataType() { - return DataType.BOOLEAN; - } - - @Override - public Nullability nullable() { - return Nullability.FALSE; - } - - @Override - protected MvContainsAll replaceChildren(Expression newLeft, Expression newRight) { - return new MvContainsAll(source(), newLeft, newRight); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, MvContainsAll::new, left(), right()); - } - - @Override - public Object fold(FoldContext ctx) { - return EvaluatorMapper.super.fold(source(), ctx); - } - - @Override - public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - var supersetType = PlannerUtils.toElementType(left().dataType()); - var subsetType = PlannerUtils.toElementType(right().dataType()); - if (supersetType != ElementType.NULL && subsetType != ElementType.NULL && supersetType != subsetType) { - throw new EsqlIllegalArgumentException( - "Incompatible data types for MvContainsAll, superset type({}) value({}) and subset type({}) value({}) don't match.", - supersetType, - left(), - subsetType, - right() - ); - } - if (supersetType == ElementType.NULL || subsetType == ElementType.NULL) { - return new MvContainsAllNullEvaluator(toEvaluator.apply(right())); - } - return switch (supersetType) { - case BOOLEAN -> new MvContainsAllBooleanEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - case BYTES_REF -> new MvContainsAllBytesRefEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - case DOUBLE -> new MvContainsAllDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - case INT -> new MvContainsAllIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - case LONG -> new MvContainsAllLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - default -> throw EsqlIllegalArgumentException.illegalDataType(dataType()); - }; - } - - @Evaluator(extraName = "Int") - static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { - appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt)); - } - - @Evaluator(extraName = "Boolean") - static void process(BooleanBlock.Builder builder, int position, BooleanBlock field1, BooleanBlock field2) { - appendTo(builder, containsAll(field1, field2, position, BooleanBlock::getBoolean)); - } - - @Evaluator(extraName = "Long") - static void process(BooleanBlock.Builder builder, int position, LongBlock field1, LongBlock field2) { - appendTo(builder, containsAll(field1, field2, position, LongBlock::getLong)); - } - - @Evaluator(extraName = "Double") - static void process(BooleanBlock.Builder builder, int position, DoubleBlock field1, DoubleBlock field2) { - appendTo(builder, containsAll(field1, field2, position, DoubleBlock::getDouble)); - } - - @Evaluator(extraName = "BytesRef") - static void process(BooleanBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) { - appendTo(builder, containsAll(field1, field2, position, (block, index) -> { - var ref = new BytesRef(); - // we pass in a reference, but sometimes we only get a return value, see ConstantBytesRefVector.getBytesRef - ref = block.getBytesRef(index, ref); - // pass empty ref as null - if (ref.length == 0) { - return null; - } - return ref; - })); - } - - static void appendTo(BooleanBlock.Builder builder, Boolean bool) { - if (bool == null) { - builder.appendNull(); - } else { - builder.beginPositionEntry().appendBoolean(bool).endPositionEntry(); - } - } - - /** - * A block is considered a subset if the superset contains values that test equal for all the values in the subset, independent of - * order. Duplicates are ignored in the sense that for each duplicate in the subset, we will search/match against the first/any value - * in the superset. - * - * @param superset block to check against - * @param subset block containing values that should be present in the other block. - * @return {@code true} if the given blocks are a superset and subset to each other, {@code false} if not. - */ - static Boolean containsAll( - BlockType superset, - BlockType subset, - final int position, - ValueExtractor valueExtractor - ) { - if (superset == subset) { - return true; - } - if (subset.areAllValuesNull()) { - return true; - } - - final var subsetCount = subset.getValueCount(position); - final var startIndex = subset.getFirstValueIndex(position); - for (int subsetIndex = startIndex; subsetIndex < startIndex + subsetCount; subsetIndex++) { - var value = valueExtractor.extractValue(subset, subsetIndex); - if (hasValue(superset, position, value, valueExtractor) == false) { - return false; - } - } - return true; - } - - /** - * Check if the block has the value at any of it's positions - * @param superset Block to search - * @param value to search for - * @return true if the supplied long value is in the supplied Block - */ - static boolean hasValue( - BlockType superset, - final int position, - Type value, - ValueExtractor valueExtractor - ) { - final var supersetCount = superset.getValueCount(position); - final var startIndex = superset.getFirstValueIndex(position); - for (int supersetIndex = startIndex; supersetIndex < startIndex + supersetCount; supersetIndex++) { - var element = valueExtractor.extractValue(superset, supersetIndex); - if (element != null && element.equals(value)) { - return true; - } - } - return false; - } - - interface ValueExtractor { - Type extractValue(BlockType block, int position); - } - - private record MvContainsAllNullEvaluator(ExpressionEvaluator.Factory subsetFieldEvaluator) implements ExpressionEvaluator.Factory { - - @Override - public ExpressionEvaluator get(DriverContext context) { - return new ExpressionEvaluator() { - final ExpressionEvaluator subsetField = subsetFieldEvaluator.get(context); - - @Override - public Block eval(Page page) { - try (Block block = subsetField.eval(page)) { - var position = page.getPositionCount(); - return context.blockFactory().newConstantBooleanBlockWith(block.isNull(position), position); - } - } - - @Override - public void close() { - Releasables.closeExpectNoException(subsetField); - } - - @Override - public String toString() { - return "MvContainsAllNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; - } - }; - } - - @Override - public String toString() { - return "MvContainsAllNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; - } - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java index 680542fc8e462..8dafc630e0e02 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFunctionWritables.java @@ -17,7 +17,7 @@ public static List getNamedWriteables() { MvAppend.ENTRY, MvAvg.ENTRY, MvConcat.ENTRY, - MvContainsAll.ENTRY, + MvContains.ENTRY, MvCount.ENTRY, MvDedupe.ENTRY, MvFirst.ENTRY, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java similarity index 86% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllErrorTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java index 4c6bafdf8ed84..c64cb8eef8d6f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java @@ -19,15 +19,15 @@ import static org.hamcrest.Matchers.equalTo; -public class MvContainsAllErrorTests extends ErrorsForCasesWithoutExamplesTestCase { +public class MvContainsErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @Override protected List cases() { - return paramsToSuppliers(MvContainsAllTests.parameters()); + return paramsToSuppliers(MvContainsTests.parameters()); } @Override protected Expression build(Source source, List args) { - return new MvContainsAll(source, args.get(0), args.get(1)); + return new MvContains(source, args.get(0), args.get(1)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsSerializationTests.java similarity index 75% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllSerializationTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsSerializationTests.java index 8663d3cc9c574..3f190b0250671 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsSerializationTests.java @@ -13,17 +13,17 @@ import java.io.IOException; -public class MvContainsAllSerializationTests extends AbstractExpressionSerializationTests { +public class MvContainsSerializationTests extends AbstractExpressionSerializationTests { @Override - protected MvContainsAll createTestInstance() { + protected MvContains createTestInstance() { Source source = randomSource(); Expression field1 = randomChild(); Expression field2 = randomChild(); - return new MvContainsAll(source, field1, field2); + return new MvContains(source, field1, field2); } @Override - protected MvContainsAll mutateInstance(MvContainsAll instance) throws IOException { + protected MvContains mutateInstance(MvContains instance) throws IOException { Source source = randomSource(); Expression field1 = randomChild(); Expression field2 = randomChild(); @@ -32,6 +32,6 @@ protected MvContainsAll mutateInstance(MvContainsAll instance) throws IOExceptio } else { field2 = randomValueOtherThan(field2, AbstractExpressionSerializationTests::randomChild); } - return new MvContainsAll(source, field1, field2); + return new MvContains(source, field1, field2); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java similarity index 79% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index dc85e313b98cd..5cee66c4a6543 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAllTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -22,10 +22,13 @@ import org.hamcrest.Matcher; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; @@ -35,8 +38,8 @@ import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; -public class MvContainsAllTests extends AbstractScalarFunctionTestCase { - public MvContainsAllTests(@Name("TestCase") Supplier testCaseSupplier) { +public class MvContainsTests extends AbstractScalarFunctionTestCase { + public MvContainsTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -49,20 +52,16 @@ public static Iterable parameters() { doubles(suppliers); bytesRefs(suppliers); - return parameterSuppliersFromTypedData( - anyNullIsNull( - suppliers, - (nullPosition, nullValueDataType, original) -> false && nullValueDataType == DataType.NULL && original.getData().size() == 1 - ? DataType.NULL - : original.expectedType(), - (nullPosition, nullData, original) -> original - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull( + suppliers, + (nullPosition, nullValueDataType, original) -> original.expectedType(), + (nullPosition, nullData, original) -> original + )); } @Override protected Expression build(Source source, List args) { - return new MvContainsAll(source, args.get(0), args.get(1)); + return new MvContains(source, args.get(0), args.get(1)); } private static void booleans(List suppliers) { @@ -75,7 +74,7 @@ private static void booleans(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.BOOLEAN, "field1"), new TestCaseSupplier.TypedData(field2, DataType.BOOLEAN, "field2") ), - "MvContainsAllBooleanEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBooleanEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -92,7 +91,7 @@ private static void ints(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.INTEGER, "field1"), new TestCaseSupplier.TypedData(field2, DataType.INTEGER, "field2") ), - "MvContainsAllIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -109,7 +108,7 @@ private static void longs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.LONG, "field1"), new TestCaseSupplier.TypedData(field2, DataType.LONG, "field2") ), - "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -123,7 +122,7 @@ private static void longs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.UNSIGNED_LONG, "field1"), new TestCaseSupplier.TypedData(field2, DataType.UNSIGNED_LONG, "field2") ), - "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -137,7 +136,7 @@ private static void longs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.DATETIME, "field1"), new TestCaseSupplier.TypedData(field2, DataType.DATETIME, "field2") ), - "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -151,7 +150,7 @@ private static void longs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.DATE_NANOS, "field1"), new TestCaseSupplier.TypedData(field2, DataType.DATE_NANOS, "field2") ), - "MvContainsAllLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -168,7 +167,7 @@ private static void doubles(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.DOUBLE, "field1"), new TestCaseSupplier.TypedData(field2, DataType.DOUBLE, "field2") ), - "MvContainsAllDoubleEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsDoubleEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -187,7 +186,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, lhs, "field1"), new TestCaseSupplier.TypedData(field2, rhs, "field2") ), - "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -203,7 +202,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.IP, "field"), new TestCaseSupplier.TypedData(field2, DataType.IP, "field") ), - "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -218,7 +217,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.VERSION, "field"), new TestCaseSupplier.TypedData(field2, DataType.VERSION, "field") ), - "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -233,7 +232,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.GEO_POINT, "field1"), new TestCaseSupplier.TypedData(field2, DataType.GEO_POINT, "field2") ), - "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -248,7 +247,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_POINT, "field1"), new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_POINT, "field2") ), - "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -263,7 +262,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.GEO_SHAPE, "field1"), new TestCaseSupplier.TypedData(field2, DataType.GEO_SHAPE, "field2") ), - "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -278,7 +277,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_SHAPE, "field1"), new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_SHAPE, "field2") ), - "MvContainsAllBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvContainsBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", DataType.BOOLEAN, equalTo(result) ); @@ -291,8 +290,7 @@ protected static List anyNullIsNull( ExpectedType expectedType, ExpectedEvaluatorToString evaluatorToString ) { - List suppliers = new ArrayList<>(testCaseSuppliers.size()); - suppliers.addAll(testCaseSuppliers); + List suppliers = new ArrayList<>(testCaseSuppliers); /* * For each original test case, add as many copies as there were @@ -309,15 +307,16 @@ protected static List anyNullIsNull( for (int typeIndex = 0; typeIndex < original.types().size(); typeIndex++) { int nullPosition = typeIndex; - suppliers.add(new TestCaseSupplier(original.name() + " null in " + nullPosition, original.types(), () -> { + suppliers.add(new TestCaseSupplier("G1: " + original.name() + " null in " + nullPosition, original.types(), () -> { TestCaseSupplier.TestCase originalTestCase = original.get(); - List data = new ArrayList<>(originalTestCase.getData()); - data.set(nullPosition, NULL); + List typeDataWithNull = new ArrayList<>(originalTestCase.getData()); + var data = typeDataWithNull.get(nullPosition); + typeDataWithNull.set(nullPosition, data.withData(data.isMultiRow() ? Collections.singletonList(null) : null)); TestCaseSupplier.TypedData nulledData = originalTestCase.getData().get(nullPosition); return new TestCaseSupplier.TestCase( - data, + typeDataWithNull, evaluatorToString.evaluatorToString(nullPosition, nulledData, originalTestCase.evaluatorToString()), - expectedType.expectedType(nullPosition, nulledData.type(), originalTestCase), + expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), equalTo(nullPosition == 1) ); })); @@ -327,17 +326,21 @@ protected static List anyNullIsNull( typesWithNull.set(nullPosition, DataType.NULL); boolean newSignature = uniqueSignatures.add(typesWithNull); if (newSignature) { - suppliers.add(new TestCaseSupplier(typesWithNull, () -> { - TestCaseSupplier.TestCase originalTestCase = original.get(); - var typeDataWithNull = new ArrayList<>(originalTestCase.getData()); - typeDataWithNull.set(nullPosition, typeDataWithNull.get(nullPosition).isMultiRow() ? MULTI_ROW_NULL : NULL); - return new TestCaseSupplier.TestCase( - typeDataWithNull, - "MvContainsAllNullEvaluator[subsetField=Attribute[channel=1]]", - DataType.BOOLEAN, - equalTo(nullPosition == 1) - ); - })); + suppliers.add(new TestCaseSupplier( + "G2: " + toSpaceSeparatedString(typesWithNull) + " null in " + nullPosition, + typesWithNull, + () -> { + TestCaseSupplier.TestCase originalTestCase = original.get(); + var typeDataWithNull = new ArrayList<>(originalTestCase.getData()); + typeDataWithNull.set(nullPosition, typeDataWithNull.get(nullPosition).isMultiRow() ? MULTI_ROW_NULL : NULL); + return new TestCaseSupplier.TestCase( + typeDataWithNull, + "MvContainsNullEvaluator[subsetField=Attribute[channel=1]]", + expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), + equalTo(nullPosition == 1) + ); + } + )); } } } @@ -346,9 +349,13 @@ protected static List anyNullIsNull( return suppliers; } + private static String toSpaceSeparatedString(ArrayList typesWithNull) { + return typesWithNull.stream().map(Objects::toString).collect(Collectors.joining(" ")); + } + // We always return a boolean. @Override protected Matcher allNullsMatcher() { - return anyOf(equalTo(false), equalTo(true)); + return anyOf(equalTo(false),equalTo(true)); } } From fbe8a2bb2f1bd10ff479bfdae1d7260236c7a6c7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 24 Aug 2025 12:11:38 +0000 Subject: [PATCH 19/41] [CI] Auto commit changes from spotless --- .../scalar/multivalue/MvContains.java | 87 ++++++++++++------- .../scalar/multivalue/MvContainsTests.java | 49 ++++++----- 2 files changed, 84 insertions(+), 52 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index ea85b01249466..993ced3520bb9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -174,7 +174,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { right() ); } - if(supersetType == ElementType.NULL || subsetType == ElementType.NULL) { + if (supersetType == ElementType.NULL || subsetType == ElementType.NULL) { return new MvContainsNullEvaluator(toEvaluator.apply(right())); } return switch (supersetType) { @@ -342,9 +342,11 @@ public static class MvContainsBooleanEvaluator implements EvalOperator.Expressio private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; - public MvContainsBooleanEvaluator(EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, - DriverContext driverContext) { + public MvContainsBooleanEvaluator( + EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext + ) { this.field1 = field1; this.field2 = field2; this.driverContext = driverContext; @@ -360,7 +362,7 @@ public Block eval(Page page) { } public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { for (int p = 0; p < positionCount; p++) { MvContains.process(result, p, field1Block, field2Block); } @@ -383,8 +385,11 @@ public static class Factory implements EvalOperator.ExpressionEvaluator.Factory private final EvalOperator.ExpressionEvaluator.Factory field1; private final EvalOperator.ExpressionEvaluator.Factory field2; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + public Factory( + Source source, + EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2 + ) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -417,9 +422,11 @@ public static class MvContainsBytesRefEvaluator implements EvalOperator.Expressi private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; - public MvContainsBytesRefEvaluator(EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, - DriverContext driverContext) { + public MvContainsBytesRefEvaluator( + EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext + ) { this.field1 = field1; this.field2 = field2; this.driverContext = driverContext; @@ -435,7 +442,7 @@ public Block eval(Page page) { } public BooleanBlock eval(int positionCount, BytesRefBlock field1Block, BytesRefBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { for (int p = 0; p < positionCount; p++) { MvContains.process(result, p, field1Block, field2Block); } @@ -458,8 +465,11 @@ public static class Factory implements EvalOperator.ExpressionEvaluator.Factory private final EvalOperator.ExpressionEvaluator.Factory field1; private final EvalOperator.ExpressionEvaluator.Factory field2; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + public Factory( + Source source, + EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2 + ) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -492,9 +502,11 @@ public static class MvContainsDoubleEvaluator implements EvalOperator.Expression private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; - public MvContainsDoubleEvaluator(EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, - DriverContext driverContext) { + public MvContainsDoubleEvaluator( + EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext + ) { this.field1 = field1; this.field2 = field2; this.driverContext = driverContext; @@ -510,7 +522,7 @@ public Block eval(Page page) { } public BooleanBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { for (int p = 0; p < positionCount; p++) { MvContains.process(result, p, field1Block, field2Block); } @@ -533,8 +545,11 @@ public static class Factory implements EvalOperator.ExpressionEvaluator.Factory private final EvalOperator.ExpressionEvaluator.Factory field1; private final EvalOperator.ExpressionEvaluator.Factory field2; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + public Factory( + Source source, + EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2 + ) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -567,9 +582,11 @@ public static class MvContainsIntEvaluator implements EvalOperator.ExpressionEva private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; - public MvContainsIntEvaluator(EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, - DriverContext driverContext) { + public MvContainsIntEvaluator( + EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext + ) { this.field1 = field1; this.field2 = field2; this.driverContext = driverContext; @@ -585,7 +602,7 @@ public Block eval(Page page) { } public BooleanBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { for (int p = 0; p < positionCount; p++) { MvContains.process(result, p, field1Block, field2Block); } @@ -608,8 +625,11 @@ public static class Factory implements EvalOperator.ExpressionEvaluator.Factory private final EvalOperator.ExpressionEvaluator.Factory field1; private final EvalOperator.ExpressionEvaluator.Factory field2; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + public Factory( + Source source, + EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2 + ) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -642,9 +662,11 @@ public static class MvContainsLongEvaluator implements EvalOperator.ExpressionEv private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; - public MvContainsLongEvaluator(EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, - DriverContext driverContext) { + public MvContainsLongEvaluator( + EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, + DriverContext driverContext + ) { this.field1 = field1; this.field2 = field2; this.driverContext = driverContext; @@ -660,7 +682,7 @@ public Block eval(Page page) { } public BooleanBlock eval(int positionCount, LongBlock field1Block, LongBlock field2Block) { - try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { for (int p = 0; p < positionCount; p++) { MvContains.process(result, p, field1Block, field2Block); } @@ -683,8 +705,11 @@ public static class Factory implements EvalOperator.ExpressionEvaluator.Factory private final EvalOperator.ExpressionEvaluator.Factory field1; private final EvalOperator.ExpressionEvaluator.Factory field2; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + public Factory( + Source source, + EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2 + ) { this.source = source; this.field1 = field1; this.field2 = field2; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index 5cee66c4a6543..0938a0f81211e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -52,11 +52,13 @@ public static Iterable parameters() { doubles(suppliers); bytesRefs(suppliers); - return parameterSuppliersFromTypedData(anyNullIsNull( - suppliers, - (nullPosition, nullValueDataType, original) -> original.expectedType(), - (nullPosition, nullData, original) -> original - )); + return parameterSuppliersFromTypedData( + anyNullIsNull( + suppliers, + (nullPosition, nullValueDataType, original) -> original.expectedType(), + (nullPosition, nullData, original) -> original + ) + ); } @Override @@ -326,21 +328,26 @@ protected static List anyNullIsNull( typesWithNull.set(nullPosition, DataType.NULL); boolean newSignature = uniqueSignatures.add(typesWithNull); if (newSignature) { - suppliers.add(new TestCaseSupplier( - "G2: " + toSpaceSeparatedString(typesWithNull) + " null in " + nullPosition, - typesWithNull, - () -> { - TestCaseSupplier.TestCase originalTestCase = original.get(); - var typeDataWithNull = new ArrayList<>(originalTestCase.getData()); - typeDataWithNull.set(nullPosition, typeDataWithNull.get(nullPosition).isMultiRow() ? MULTI_ROW_NULL : NULL); - return new TestCaseSupplier.TestCase( - typeDataWithNull, - "MvContainsNullEvaluator[subsetField=Attribute[channel=1]]", - expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), - equalTo(nullPosition == 1) - ); - } - )); + suppliers.add( + new TestCaseSupplier( + "G2: " + toSpaceSeparatedString(typesWithNull) + " null in " + nullPosition, + typesWithNull, + () -> { + TestCaseSupplier.TestCase originalTestCase = original.get(); + var typeDataWithNull = new ArrayList<>(originalTestCase.getData()); + typeDataWithNull.set( + nullPosition, + typeDataWithNull.get(nullPosition).isMultiRow() ? MULTI_ROW_NULL : NULL + ); + return new TestCaseSupplier.TestCase( + typeDataWithNull, + "MvContainsNullEvaluator[subsetField=Attribute[channel=1]]", + expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), + equalTo(nullPosition == 1) + ); + } + ) + ); } } } @@ -356,6 +363,6 @@ private static String toSpaceSeparatedString(ArrayList typesWithNull) // We always return a boolean. @Override protected Matcher allNullsMatcher() { - return anyOf(equalTo(false),equalTo(true)); + return anyOf(equalTo(false), equalTo(true)); } } From f71848f2480f743375259fd3883c15289b9d561b Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Sun, 24 Aug 2025 15:47:41 +0200 Subject: [PATCH 20/41] Update documentation to reflect `MV_CONTAINS_ALL` renaming to `MV_CONTAINS` --- .../query-languages/esql/_snippets/lists/mv-functions.md | 2 +- .../query-languages/esql/functions-operators/mv-functions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md b/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md index 983d886c3568f..9129b5f6fb559 100644 --- a/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md +++ b/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md @@ -1,7 +1,7 @@ * [`MV_APPEND`](../../functions-operators/mv-functions.md#esql-mv_append) * [`MV_AVG`](../../functions-operators/mv-functions.md#esql-mv_avg) * [`MV_CONCAT`](../../functions-operators/mv-functions.md#esql-mv_concat) -* [`MV_CONTAINS_ALL`](../../functions-operators/mv-functions.md#esql-mv_contains_all) +* [`MV_CONTAINS`](../../functions-operators/mv-functions.md#esql-mv_contains) * [`MV_COUNT`](../../functions-operators/mv-functions.md#esql-mv_count) * [`MV_DEDUPE`](../../functions-operators/mv-functions.md#esql-mv_dedupe) * [`MV_FIRST`](../../functions-operators/mv-functions.md#esql-mv_first) diff --git a/docs/reference/query-languages/esql/functions-operators/mv-functions.md b/docs/reference/query-languages/esql/functions-operators/mv-functions.md index 81c542b9ade1d..acb0b882e6bfc 100644 --- a/docs/reference/query-languages/esql/functions-operators/mv-functions.md +++ b/docs/reference/query-languages/esql/functions-operators/mv-functions.md @@ -21,7 +21,7 @@ mapped_pages: :::{include} ../_snippets/functions/layout/mv_concat.md ::: -:::{include} ../_snippets/functions/layout/mv_contains_all.md +:::{include} ../_snippets/functions/layout/mv_contains.md ::: :::{include} ../_snippets/functions/layout/mv_count.md From 5584a937ef0c78ad8901622bdaeffc5df94d8f9f Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Mon, 25 Aug 2025 17:34:24 +0200 Subject: [PATCH 21/41] Fix `MvContains` javadoc --- .../scalar/multivalue/MvContains.java | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 993ced3520bb9..5396c17415701 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -46,7 +46,43 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; /** - * Reduce a multivalued field to a single valued field containing the count of values. + * Function that takes two multi valued expressions and checks if values of one expression are all present(equals) in the other. + *
    + *
  • Set A = {"a","b","c"}
    • + *
  • Set B = {"b","c"}
    • + *
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
+ * Second Argument
A B null
First
Argument		 A true true true
B false true true
null false false true
+ * */ public class MvContains extends BinaryScalarFunction implements EvaluatorMapper { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( From bfa03d829ccaa29dff5eb701a20ca2deecd8e90e Mon Sep 17 00:00:00 2001 From: Michael Bischoff Date: Mon, 25 Aug 2025 17:36:22 +0200 Subject: [PATCH 22/41] Add preview marker to documentation link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Iván Cea Fontenla --- .../query-languages/esql/_snippets/lists/mv-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md b/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md index 9129b5f6fb559..db2d1149e7f75 100644 --- a/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md +++ b/docs/reference/query-languages/esql/_snippets/lists/mv-functions.md @@ -1,7 +1,7 @@ * [`MV_APPEND`](../../functions-operators/mv-functions.md#esql-mv_append) * [`MV_AVG`](../../functions-operators/mv-functions.md#esql-mv_avg) * [`MV_CONCAT`](../../functions-operators/mv-functions.md#esql-mv_concat) -* [`MV_CONTAINS`](../../functions-operators/mv-functions.md#esql-mv_contains) +* [preview] [`MV_CONTAINS`](../../functions-operators/mv-functions.md#esql-mv_contains) * [`MV_COUNT`](../../functions-operators/mv-functions.md#esql-mv_count) * [`MV_DEDUPE`](../../functions-operators/mv-functions.md#esql-mv_dedupe) * [`MV_FIRST`](../../functions-operators/mv-functions.md#esql-mv_first) From b9a222b6d31c71b44a92c1ebb1fc073cc11a5f97 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Mon, 25 Aug 2025 18:15:02 +0200 Subject: [PATCH 23/41] I think checkstyle needs to be updated :D --- .../scalar/multivalue/MvContains.java | 76 +++++++++---------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 5396c17415701..77bbdb3bd6e81 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -45,45 +45,43 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -/** - * Function that takes two multi valued expressions and checks if values of one expression are all present(equals) in the other. - *
    - *
  • Set A = {"a","b","c"}
    • - *
  • Set B = {"b","c"}
    • - *
- * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
- * Second Argument
A B null
First
Argument		 A true true true
B false true true
null false false true
- * - */ +/// Function that takes two multivalued expressions and checks if values of one expression are all present(equals) in the other. +/// +/// - Set A = {"a","b","c"} +/// - Set B = {"b","c"} +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +///
+/// Second Argument
A B null
First +/// Argument A true true true
B false true true
null false false true
public class MvContains extends BinaryScalarFunction implements EvaluatorMapper { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, From 445e7bb087d30c0cb2caa59c67a39e236d23e46f Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Mon, 25 Aug 2025 20:08:09 +0200 Subject: [PATCH 24/41] Replacing table and markdown comment block --- .../scalar/multivalue/MvContains.java | 53 ++++++------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 77bbdb3bd6e81..2b429da17e9d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -45,43 +45,22 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -/// Function that takes two multivalued expressions and checks if values of one expression are all present(equals) in the other. -/// -/// - Set A = {"a","b","c"} -/// - Set B = {"b","c"} -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -/// -///
-/// Second Argument
A B null
First -/// Argument A true true true
B false true true
null false false true
+/** + * Function that takes two multivalued expressions and checks if values of one expression are all present(equals) in the other. + *

+ * Given Set A = {"a","b","c"} and Set B = {"b","c"}, the relationship between first (row) and second (column) arguments is: + *

    + *
  • A, B ⇒ true (A ⊆ B)
    • + *
  • B, A ⇒ false (A ⊈ B)
    • + *
  • A, A ⇒ true (A ≡ A)
    • + *
  • B, B ⇒ true (B ≡ B)
    • + *
  • A, null ⇒ true (B ⊆ ∅)
    • + *
  • null, A ⇒ false (∅ ⊈ B)
    • + *
  • B, null ⇒ true (B ⊆ ∅)
    • + *
  • null, B ⇒ false (∅ ⊈ B)
    • + *
  • null, null ⇒ true (∅ ≡ ∅)
    • + *
+ */ public class MvContains extends BinaryScalarFunction implements EvaluatorMapper { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, From 277249a6d125fbb76b67270530756b64c7e96c97 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Wed, 27 Aug 2025 12:29:43 +0200 Subject: [PATCH 25/41] Add memory usage reporting to `MvContains` evaluators ESQL: Track memory in evaluators (#133392) got merged to main at the same as Add MV_CONTAINS function #133099 which caused a compile-error and the merge was reverted. This commit addresses the compile-error. --- .../scalar/multivalue/MvContains.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 2b429da17e9d6..1b78b9433fd1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.data.Block; @@ -303,6 +304,7 @@ interface ValueExtractor { } private static final class MvContainsNullEvaluator implements ExpressionEvaluator.Factory { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsNullEvaluator.class); private final ExpressionEvaluator.Factory subsetFieldEvaluator; private MvContainsNullEvaluator(ExpressionEvaluator.Factory subsetFieldEvaluator) { @@ -322,6 +324,13 @@ public Block eval(Page page) { } } + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += subsetField.baseRamBytesUsed(); + return baseRamBytesUsed; + } + @Override public void close() { Releasables.closeExpectNoException(subsetField); @@ -351,6 +360,7 @@ public String toString() { * TODO extend code generation to handle this case */ public static class MvContainsBooleanEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsBooleanEvaluator.class); private final EvalOperator.ExpressionEvaluator field1; private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; @@ -393,6 +403,14 @@ public void close() { Releasables.closeExpectNoException(field1, field2); } + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += field1.baseRamBytesUsed(); + baseRamBytesUsed += field2.baseRamBytesUsed(); + return baseRamBytesUsed; + } + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; private final EvalOperator.ExpressionEvaluator.Factory field1; @@ -431,6 +449,7 @@ public String toString() { * TODO extend code generation to handle this case */ public static class MvContainsBytesRefEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsBytesRefEvaluator.class); private final EvalOperator.ExpressionEvaluator field1; private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; @@ -473,6 +492,14 @@ public void close() { Releasables.closeExpectNoException(field1, field2); } + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += field1.baseRamBytesUsed(); + baseRamBytesUsed += field2.baseRamBytesUsed(); + return baseRamBytesUsed; + } + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; private final EvalOperator.ExpressionEvaluator.Factory field1; @@ -511,6 +538,7 @@ public String toString() { * TODO extend code generation to handle this case */ public static class MvContainsDoubleEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsDoubleEvaluator.class); private final EvalOperator.ExpressionEvaluator field1; private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; @@ -553,6 +581,14 @@ public void close() { Releasables.closeExpectNoException(field1, field2); } + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += field1.baseRamBytesUsed(); + baseRamBytesUsed += field2.baseRamBytesUsed(); + return baseRamBytesUsed; + } + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; private final EvalOperator.ExpressionEvaluator.Factory field1; @@ -591,6 +627,7 @@ public String toString() { * TODO extend code generation to handle this case */ public static class MvContainsIntEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsIntEvaluator.class); private final EvalOperator.ExpressionEvaluator field1; private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; @@ -633,6 +670,14 @@ public void close() { Releasables.closeExpectNoException(field1, field2); } + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += field1.baseRamBytesUsed(); + baseRamBytesUsed += field2.baseRamBytesUsed(); + return baseRamBytesUsed; + } + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; private final EvalOperator.ExpressionEvaluator.Factory field1; @@ -671,6 +716,7 @@ public String toString() { * TODO extend code generation to handle this case */ public static class MvContainsLongEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsLongEvaluator.class); private final EvalOperator.ExpressionEvaluator field1; private final EvalOperator.ExpressionEvaluator field2; private final DriverContext driverContext; @@ -713,6 +759,14 @@ public void close() { Releasables.closeExpectNoException(field1, field2); } + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += field1.baseRamBytesUsed(); + baseRamBytesUsed += field2.baseRamBytesUsed(); + return baseRamBytesUsed; + } + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; private final EvalOperator.ExpressionEvaluator.Factory field1; From a8563602bc49b8466ed4fd2004bdf91a32d55991 Mon Sep 17 00:00:00 2001 From: Michael Bischoff Date: Wed, 27 Aug 2025 12:37:53 +0200 Subject: [PATCH 26/41] Update docs/changelog/133636.yaml --- docs/changelog/133636.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/133636.yaml diff --git a/docs/changelog/133636.yaml b/docs/changelog/133636.yaml new file mode 100644 index 0000000000000..46202e2d1c2b1 --- /dev/null +++ b/docs/changelog/133636.yaml @@ -0,0 +1,5 @@ +pr: 133636 +summary: Esql `mv_contains` function +area: ES|QL +type: enhancement +issues: [] From f4d3020b43471860f886eafcfc175d9463983bd0 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Thu, 28 Aug 2025 04:32:13 +0200 Subject: [PATCH 27/41] removing yaml from other PR --- docs/changelog/133099.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/133099.yaml diff --git a/docs/changelog/133099.yaml b/docs/changelog/133099.yaml deleted file mode 100644 index 8e81128073787..0000000000000 --- a/docs/changelog/133099.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 133099 -summary: Add MV_CONTAINS function -area: ES|QL -type: enhancement -issues: [] From 1f7f430a7769fb989190e9ff97e2337bb2212e71 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Thu, 28 Aug 2025 08:13:21 +0200 Subject: [PATCH 28/41] WIP still has test failures. Refactor null handling in `MvContains` evaluators and add `MvContainsNullSupersetEvaluator` for better type-specific evaluation logic. --- .../scalar/multivalue/MvContains.java | 145 ++++++++++++++---- .../scalar/multivalue/MvContainsTests.java | 4 +- 2 files changed, 116 insertions(+), 33 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 1b78b9433fd1f..61aa2788a61ee 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -179,7 +179,12 @@ public Object fold(FoldContext ctx) { public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var supersetType = PlannerUtils.toElementType(left().dataType()); var subsetType = PlannerUtils.toElementType(right().dataType()); - if (supersetType != ElementType.NULL && subsetType != ElementType.NULL && supersetType != subsetType) { + + if (subsetType == ElementType.NULL) { + return new ConstantBooleanTrueEvaluator(); + } + + if (supersetType != ElementType.NULL && supersetType != subsetType) { throw new EsqlIllegalArgumentException( "Incompatible data types for MvContains, superset type({}) value({}) and subset type({}) value({}) don't match.", supersetType, @@ -188,19 +193,31 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { right() ); } - if (supersetType == ElementType.NULL || subsetType == ElementType.NULL) { - return new MvContainsNullEvaluator(toEvaluator.apply(right())); - } + return switch (supersetType) { case BOOLEAN -> new MvContainsBooleanEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case BYTES_REF -> new MvContainsBytesRefEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case DOUBLE -> new MvContainsDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case INT -> new MvContainsIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case LONG -> new MvContainsLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); + case NULL -> new MvContainsNullSupersetEvaluator.Factory(source(), toEvaluator.apply(right())); default -> throw EsqlIllegalArgumentException.illegalDataType(dataType()); }; } + static void process(BooleanBlock.Builder builder, int position, Block subset) { + final var valueCount = subset.getValueCount(position); + final var startIndex = subset.getFirstValueIndex(position); + for (int valueIndex = startIndex; valueIndex < startIndex + valueCount; valueIndex++) { + if(subset.isNull(valueIndex)) { + continue; + } + appendTo(builder, false); + return; + } + appendTo(builder, true); + } + // @Evaluator(extraName = "Int") see end of file. static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt)); @@ -265,10 +282,13 @@ static Boolean containsAll( return true; } - final var subsetCount = subset.getValueCount(position); + final var valueCount = subset.getValueCount(position); final var startIndex = subset.getFirstValueIndex(position); - for (int subsetIndex = startIndex; subsetIndex < startIndex + subsetCount; subsetIndex++) { - var value = valueExtractor.extractValue(subset, subsetIndex); + for (int valueIndex = startIndex; valueIndex < startIndex + valueCount; valueIndex++) { + var value = valueExtractor.extractValue(subset, valueIndex); + if(value == null) { // null entries are considered to always be an element in the superset. + continue; + } if (hasValue(superset, position, value, valueExtractor) == false) { return false; } @@ -303,49 +323,110 @@ interface ValueExtractor { Type extractValue(BlockType block, int position); } - private static final class MvContainsNullEvaluator implements ExpressionEvaluator.Factory { - private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsNullEvaluator.class); - private final ExpressionEvaluator.Factory subsetFieldEvaluator; - - private MvContainsNullEvaluator(ExpressionEvaluator.Factory subsetFieldEvaluator) { - this.subsetFieldEvaluator = subsetFieldEvaluator; - } - + /** + * Evaluator that always returns true for all values in the block (~column) + */ + public static final class ConstantBooleanTrueEvaluator implements ExpressionEvaluator.Factory { @Override - public ExpressionEvaluator get(DriverContext context) { + public ExpressionEvaluator get(DriverContext driverContext) { return new ExpressionEvaluator() { - final ExpressionEvaluator subsetField = subsetFieldEvaluator.get(context); - @Override public Block eval(Page page) { - try (Block block = subsetField.eval(page)) { - var position = page.getPositionCount(); - return context.blockFactory().newConstantBooleanBlockWith(block.isNull(position), position); - } + return driverContext.blockFactory().newConstantBooleanBlockWith(true, page.getPositionCount()); } @Override - public long baseRamBytesUsed() { - long baseRamBytesUsed = BASE_RAM_BYTES_USED; - baseRamBytesUsed += subsetField.baseRamBytesUsed(); - return baseRamBytesUsed; - } + public void close() {} @Override - public void close() { - Releasables.closeExpectNoException(subsetField); + public String toString() { + return "ConstantBooleanTrueEvaluator"; } @Override - public String toString() { - return "MvContainsNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + public long baseRamBytesUsed() { + return 0; } }; } @Override public String toString() { - return "MvContainsNullEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + return "ConstantBooleanTrueEvaluator"; + } + } + + /** + * Evaluator for when the lhs is null + * Like the ones below should be able to autogenerate as well when the generator is more flexible + */ + public static class MvContainsNullSupersetEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsNullSupersetEvaluator.class); + private final EvalOperator.ExpressionEvaluator subsetFieldEvaluator; + private final DriverContext driverContext; + + public MvContainsNullSupersetEvaluator( + EvalOperator.ExpressionEvaluator subsetFieldEvaluator, + DriverContext driverContext + ) { + this.subsetFieldEvaluator = subsetFieldEvaluator; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (Block block = subsetFieldEvaluator.eval(page)) { + return eval(page.getPositionCount(), block); + } + } + + public BooleanBlock eval(int positionCount, Block subsetBlock) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + MvContains.process(result, p, subsetBlock); + } + return result.build(); + } + } + + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += subsetFieldEvaluator.baseRamBytesUsed(); + return baseRamBytesUsed; + } + + @Override + public void close() { + Releasables.closeExpectNoException(subsetFieldEvaluator); + } + + @Override + public String toString() { + return "MvContainsNullSupersetEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory subsetFieldEvaluator; + + public Factory( + Source source, + EvalOperator.ExpressionEvaluator.Factory subsetField + ) { + this.source = source; + this.subsetFieldEvaluator = subsetField; + } + + @Override + public MvContainsNullSupersetEvaluator get(DriverContext context) { + return new MvContainsNullSupersetEvaluator(subsetFieldEvaluator.get(context), context); + } + + @Override + public String toString() { + return "MvContainsNullSupersetEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index 0938a0f81211e..624e2994349dc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -341,7 +341,9 @@ protected static List anyNullIsNull( ); return new TestCaseSupplier.TestCase( typeDataWithNull, - "MvContainsNullEvaluator[subsetField=Attribute[channel=1]]", + nullPosition == 0 ? + "MvContainsNullSupersetEvaluator[subsetField=Attribute[channel=1]]" : + "ConstantBooleanTrueEvaluator", expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), equalTo(nullPosition == 1) ); From 2393634b74191f37cf970d1c54a8967347899ea1 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 28 Aug 2025 06:19:47 +0000 Subject: [PATCH 29/41] [CI] Auto commit changes from spotless --- .../function/scalar/multivalue/MvContains.java | 14 ++++---------- .../scalar/multivalue/MvContainsTests.java | 6 +++--- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 61aa2788a61ee..6a7e677c72902 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -209,7 +209,7 @@ static void process(BooleanBlock.Builder builder, int position, Block subset) { final var valueCount = subset.getValueCount(position); final var startIndex = subset.getFirstValueIndex(position); for (int valueIndex = startIndex; valueIndex < startIndex + valueCount; valueIndex++) { - if(subset.isNull(valueIndex)) { + if (subset.isNull(valueIndex)) { continue; } appendTo(builder, false); @@ -286,7 +286,7 @@ static Boolean containsAll( final var startIndex = subset.getFirstValueIndex(position); for (int valueIndex = startIndex; valueIndex < startIndex + valueCount; valueIndex++) { var value = valueExtractor.extractValue(subset, valueIndex); - if(value == null) { // null entries are considered to always be an element in the superset. + if (value == null) { // null entries are considered to always be an element in the superset. continue; } if (hasValue(superset, position, value, valueExtractor) == false) { @@ -365,10 +365,7 @@ public static class MvContainsNullSupersetEvaluator implements EvalOperator.Expr private final EvalOperator.ExpressionEvaluator subsetFieldEvaluator; private final DriverContext driverContext; - public MvContainsNullSupersetEvaluator( - EvalOperator.ExpressionEvaluator subsetFieldEvaluator, - DriverContext driverContext - ) { + public MvContainsNullSupersetEvaluator(EvalOperator.ExpressionEvaluator subsetFieldEvaluator, DriverContext driverContext) { this.subsetFieldEvaluator = subsetFieldEvaluator; this.driverContext = driverContext; } @@ -410,10 +407,7 @@ public static class Factory implements EvalOperator.ExpressionEvaluator.Factory private final Source source; private final EvalOperator.ExpressionEvaluator.Factory subsetFieldEvaluator; - public Factory( - Source source, - EvalOperator.ExpressionEvaluator.Factory subsetField - ) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory subsetField) { this.source = source; this.subsetFieldEvaluator = subsetField; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index 624e2994349dc..5fa897b7f262a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -341,9 +341,9 @@ protected static List anyNullIsNull( ); return new TestCaseSupplier.TestCase( typeDataWithNull, - nullPosition == 0 ? - "MvContainsNullSupersetEvaluator[subsetField=Attribute[channel=1]]" : - "ConstantBooleanTrueEvaluator", + nullPosition == 0 + ? "MvContainsNullSupersetEvaluator[subsetField=Attribute[channel=1]]" + : "ConstantBooleanTrueEvaluator", expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), equalTo(nullPosition == 1) ); From b3da482a8777a16e3eee222cb93e8e2bf797da30 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Thu, 28 Aug 2025 16:59:45 +0200 Subject: [PATCH 30/41] It seems like the isNull operates on the row position, not the value position. --- .../expression/function/scalar/multivalue/MvContains.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 6a7e677c72902..a99114f1c951f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -206,12 +206,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } static void process(BooleanBlock.Builder builder, int position, Block subset) { - final var valueCount = subset.getValueCount(position); - final var startIndex = subset.getFirstValueIndex(position); - for (int valueIndex = startIndex; valueIndex < startIndex + valueCount; valueIndex++) { - if (subset.isNull(valueIndex)) { - continue; - } + if(subset.isNull(position) == false) { appendTo(builder, false); return; } From dcc13105ea75d2455384be52c46665c144017738 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 28 Aug 2025 15:16:41 +0000 Subject: [PATCH 31/41] [CI] Auto commit changes from spotless --- .../esql/expression/function/scalar/multivalue/MvContains.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index a99114f1c951f..238fc32c3dbdf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -206,7 +206,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } static void process(BooleanBlock.Builder builder, int position, Block subset) { - if(subset.isNull(position) == false) { + if (subset.isNull(position) == false) { appendTo(builder, false); return; } From aee4e355662097132cea3cb1083b6f5bdcaf5350 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Thu, 28 Aug 2025 20:29:09 +0200 Subject: [PATCH 32/41] Refactor `MvContains` null handling by reusing existing `IsNullEvaluatorFactory` and adding global constant boolean evaluators for improved reusability. --- .../compute/operator/EvalOperator.java | 62 ++++++++++++++++ .../scalar/multivalue/MvContains.java | 73 +------------------ .../expression/predicate/nulls/IsNull.java | 11 +-- 3 files changed, 71 insertions(+), 75 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java index 2c9bf74fb8b0a..ad5c4d4f19290 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java @@ -187,4 +187,66 @@ public String toString() { } }; private static final String CONSTANT_NULL_NAME = "ConstantNull"; + + public static final ExpressionEvaluator.Factory CONSTANT_TRUE_FACTORY = new ExpressionEvaluator.Factory() { + @Override + public ExpressionEvaluator get(DriverContext driverContext) { + return new ExpressionEvaluator() { + @Override + public Block eval(Page page) { + return driverContext.blockFactory().newConstantBooleanBlockWith(true, page.getPositionCount()); + } + + @Override + public void close() {} + + @Override + public String toString() { + return CONSTANT_TRUE_NAME; + } + + @Override + public long baseRamBytesUsed() { + return 0; + } + }; + } + + @Override + public String toString() { + return CONSTANT_TRUE_NAME; + } + }; + private static final String CONSTANT_TRUE_NAME = "ConstantTrue"; + + public static final ExpressionEvaluator.Factory CONSTANT_FALSE_FACTORY = new ExpressionEvaluator.Factory() { + @Override + public ExpressionEvaluator get(DriverContext driverContext) { + return new ExpressionEvaluator() { + @Override + public Block eval(Page page) { + return driverContext.blockFactory().newConstantBooleanBlockWith(false, page.getPositionCount()); + } + + @Override + public void close() {} + + @Override + public String toString() { + return CONSTANT_FALSE_NAME; + } + + @Override + public long baseRamBytesUsed() { + return 0; + } + }; + } + + @Override + public String toString() { + return CONSTANT_FALSE_NAME; + } + }; + private static final String CONSTANT_FALSE_NAME = "ConstantFalse"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 238fc32c3dbdf..8f1be8a924e34 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import java.io.IOException; @@ -181,7 +182,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var subsetType = PlannerUtils.toElementType(right().dataType()); if (subsetType == ElementType.NULL) { - return new ConstantBooleanTrueEvaluator(); + return EvalOperator.CONSTANT_TRUE_FACTORY; } if (supersetType != ElementType.NULL && supersetType != subsetType) { @@ -200,7 +201,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { case DOUBLE -> new MvContainsDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case INT -> new MvContainsIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); case LONG -> new MvContainsLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right())); - case NULL -> new MvContainsNullSupersetEvaluator.Factory(source(), toEvaluator.apply(right())); + case NULL -> new IsNull.IsNullEvaluatorFactory(toEvaluator.apply(right())); default -> throw EsqlIllegalArgumentException.illegalDataType(dataType()); }; } @@ -351,74 +352,6 @@ public String toString() { } } - /** - * Evaluator for when the lhs is null - * Like the ones below should be able to autogenerate as well when the generator is more flexible - */ - public static class MvContainsNullSupersetEvaluator implements EvalOperator.ExpressionEvaluator { - private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsNullSupersetEvaluator.class); - private final EvalOperator.ExpressionEvaluator subsetFieldEvaluator; - private final DriverContext driverContext; - - public MvContainsNullSupersetEvaluator(EvalOperator.ExpressionEvaluator subsetFieldEvaluator, DriverContext driverContext) { - this.subsetFieldEvaluator = subsetFieldEvaluator; - this.driverContext = driverContext; - } - - @Override - public Block eval(Page page) { - try (Block block = subsetFieldEvaluator.eval(page)) { - return eval(page.getPositionCount(), block); - } - } - - public BooleanBlock eval(int positionCount, Block subsetBlock) { - try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - MvContains.process(result, p, subsetBlock); - } - return result.build(); - } - } - - @Override - public long baseRamBytesUsed() { - long baseRamBytesUsed = BASE_RAM_BYTES_USED; - baseRamBytesUsed += subsetFieldEvaluator.baseRamBytesUsed(); - return baseRamBytesUsed; - } - - @Override - public void close() { - Releasables.closeExpectNoException(subsetFieldEvaluator); - } - - @Override - public String toString() { - return "MvContainsNullSupersetEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; - } - - public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { - private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory subsetFieldEvaluator; - - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory subsetField) { - this.source = source; - this.subsetFieldEvaluator = subsetField; - } - - @Override - public MvContainsNullSupersetEvaluator get(DriverContext context) { - return new MvContainsNullSupersetEvaluator(subsetFieldEvaluator.get(context), context); - } - - @Override - public String toString() { - return "MvContainsNullSupersetEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]"; - } - } - } - /** * Currently {@code EvaluatorImplementer} generates: * if (allBlocksAreNulls) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java index 6f91c83940b34..65f5f42374a58 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -103,7 +104,7 @@ public Object fold(FoldContext ctx) { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new IsNullEvaluatorFactory(toEvaluator.apply(field())); } @@ -138,9 +139,9 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand return new NotQuery(source(), new ExistsQuery(source(), handler.nameOf(field()))); } - record IsNullEvaluatorFactory(EvalOperator.ExpressionEvaluator.Factory field) implements EvalOperator.ExpressionEvaluator.Factory { + public record IsNullEvaluatorFactory(ExpressionEvaluator.Factory field) implements ExpressionEvaluator.Factory { @Override - public EvalOperator.ExpressionEvaluator get(DriverContext context) { + public ExpressionEvaluator get(DriverContext context) { return new IsNullEvaluator(context, field.get(context)); } @@ -150,9 +151,9 @@ public String toString() { } } - record IsNullEvaluator(DriverContext driverContext, EvalOperator.ExpressionEvaluator field) + record IsNullEvaluator(DriverContext driverContext, ExpressionEvaluator field) implements - EvalOperator.ExpressionEvaluator { + ExpressionEvaluator { private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IsNullEvaluator.class); @Override From 87ec7ab1d7685ac7c25dc2766e34be02d2039086 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Thu, 28 Aug 2025 20:29:09 +0200 Subject: [PATCH 33/41] Refactor `MvContains` null handling by reusing existing `IsNullEvaluatorFactory` and adding global constant boolean evaluators for improved reusability. --- .../function/scalar/multivalue/MvContainsTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index 5fa897b7f262a..0236394eb0bfd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -342,8 +342,8 @@ protected static List anyNullIsNull( return new TestCaseSupplier.TestCase( typeDataWithNull, nullPosition == 0 - ? "MvContainsNullSupersetEvaluator[subsetField=Attribute[channel=1]]" - : "ConstantBooleanTrueEvaluator", + ? "IsNullEvaluator[field=Attribute[channel=1]]" + : "ConstantTrue", expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), equalTo(nullPosition == 1) ); From 23b2aadb64ff5c6f509a4117dbee8e4fa003801e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 28 Aug 2025 18:38:26 +0000 Subject: [PATCH 34/41] [CI] Auto commit changes from spotless --- .../xpack/esql/expression/predicate/nulls/IsNull.java | 5 +---- .../function/scalar/multivalue/MvContainsTests.java | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java index 65f5f42374a58..d0baee75f817b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java @@ -12,7 +12,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; @@ -151,9 +150,7 @@ public String toString() { } } - record IsNullEvaluator(DriverContext driverContext, ExpressionEvaluator field) - implements - ExpressionEvaluator { + record IsNullEvaluator(DriverContext driverContext, ExpressionEvaluator field) implements ExpressionEvaluator { private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IsNullEvaluator.class); @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index 0236394eb0bfd..cce33c1e76292 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -341,9 +341,7 @@ protected static List anyNullIsNull( ); return new TestCaseSupplier.TestCase( typeDataWithNull, - nullPosition == 0 - ? "IsNullEvaluator[field=Attribute[channel=1]]" - : "ConstantTrue", + nullPosition == 0 ? "IsNullEvaluator[field=Attribute[channel=1]]" : "ConstantTrue", expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase), equalTo(nullPosition == 1) ); From 4aaac356d218383927c4bdd7348b3fe8782a59b1 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Thu, 28 Aug 2025 21:44:04 +0200 Subject: [PATCH 35/41] Refactor constant evaluators in `EvalOperator` by introducing dedicated records for `ConstantNull`, `ConstantTrue`, and `ConstantFalse`, adding memory usage tracking. --- .../compute/operator/EvalOperator.java | 155 ++++++++++-------- 1 file changed, 85 insertions(+), 70 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java index ad5c4d4f19290..e8ff37f3c914b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java @@ -157,96 +157,111 @@ default boolean eagerEvalSafeInLazy() { long baseRamBytesUsed(); } - public static final ExpressionEvaluator.Factory CONSTANT_NULL_FACTORY = new ExpressionEvaluator.Factory() { + private record ConstantNullEvaluator(DriverContext context) implements ExpressionEvaluator { + private static final String NAME = "ConstantNull"; + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantNullEvaluator.class); + @Override - public ExpressionEvaluator get(DriverContext driverContext) { - return new ExpressionEvaluator() { - @Override - public Block eval(Page page) { - return driverContext.blockFactory().newConstantNullBlock(page.getPositionCount()); - } - - @Override - public void close() {} - - @Override - public String toString() { - return CONSTANT_NULL_NAME; - } - - @Override - public long baseRamBytesUsed() { - return 0; - } - }; + public Block eval(Page page) { + return context.blockFactory().newConstantNullBlock(page.getPositionCount()); } + @Override + public void close() {} + @Override public String toString() { - return CONSTANT_NULL_NAME; + return NAME; } - }; - private static final String CONSTANT_NULL_NAME = "ConstantNull"; - public static final ExpressionEvaluator.Factory CONSTANT_TRUE_FACTORY = new ExpressionEvaluator.Factory() { @Override - public ExpressionEvaluator get(DriverContext driverContext) { - return new ExpressionEvaluator() { - @Override - public Block eval(Page page) { - return driverContext.blockFactory().newConstantBooleanBlockWith(true, page.getPositionCount()); - } - - @Override - public void close() {} - - @Override - public String toString() { - return CONSTANT_TRUE_NAME; - } - - @Override - public long baseRamBytesUsed() { - return 0; - } + public long baseRamBytesUsed() { + return BASE_RAM_BYTES_USED; + } + + record Factory() implements ExpressionEvaluator.Factory { + @Override + public ConstantNullEvaluator get(DriverContext context) { + return new ConstantNullEvaluator(context); }; + + @Override + public String toString() { + return NAME; + } + }; + } + public static final ExpressionEvaluator.Factory CONSTANT_NULL_FACTORY = new ConstantNullEvaluator.Factory(); + + private record ConstantTrueEvaluator(DriverContext context) implements ExpressionEvaluator { + private static final String NAME = "ConstantTrue"; + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantTrueEvaluator.class); + + @Override + public Block eval(Page page) { + return context.blockFactory().newConstantBooleanBlockWith(true, page.getPositionCount()); } + @Override + public void close() {} + @Override public String toString() { - return CONSTANT_TRUE_NAME; + return NAME; } - }; - private static final String CONSTANT_TRUE_NAME = "ConstantTrue"; - public static final ExpressionEvaluator.Factory CONSTANT_FALSE_FACTORY = new ExpressionEvaluator.Factory() { @Override - public ExpressionEvaluator get(DriverContext driverContext) { - return new ExpressionEvaluator() { - @Override - public Block eval(Page page) { - return driverContext.blockFactory().newConstantBooleanBlockWith(false, page.getPositionCount()); - } - - @Override - public void close() {} - - @Override - public String toString() { - return CONSTANT_FALSE_NAME; - } - - @Override - public long baseRamBytesUsed() { - return 0; - } + public long baseRamBytesUsed() { + return BASE_RAM_BYTES_USED; + } + + record Factory() implements ExpressionEvaluator.Factory { + @Override + public ConstantTrueEvaluator get(DriverContext context) { + return new ConstantTrueEvaluator(context); }; + + @Override + public String toString() { + return NAME; + } + }; + } + public static final ExpressionEvaluator.Factory CONSTANT_TRUE_FACTORY = new ConstantTrueEvaluator.Factory(); + + private record ConstantFalseEvaluator(DriverContext context) implements ExpressionEvaluator { + private static final String NAME = "ConstantFalse"; + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantFalseEvaluator.class); + + @Override + public Block eval(Page page) { + return context.blockFactory().newConstantBooleanBlockWith(false, page.getPositionCount()); } + @Override + public void close() {} + @Override public String toString() { - return CONSTANT_FALSE_NAME; + return NAME; } - }; - private static final String CONSTANT_FALSE_NAME = "ConstantFalse"; + + @Override + public long baseRamBytesUsed() { + return BASE_RAM_BYTES_USED; + } + + record Factory() implements ExpressionEvaluator.Factory { + @Override + public ConstantFalseEvaluator get(DriverContext context) { + return new ConstantFalseEvaluator(context); + }; + + @Override + public String toString() { + return NAME; + } + }; + } + public static final ExpressionEvaluator.Factory CONSTANT_FALSE_FACTORY = new ConstantFalseEvaluator.Factory(); } From d676cef3d64060f59ee6fb8db338c2ca30cdf8f7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 28 Aug 2025 20:18:50 +0000 Subject: [PATCH 36/41] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/compute/operator/EvalOperator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java index e8ff37f3c914b..983b1cc7cfcbf 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java @@ -191,6 +191,7 @@ public String toString() { } }; } + public static final ExpressionEvaluator.Factory CONSTANT_NULL_FACTORY = new ConstantNullEvaluator.Factory(); private record ConstantTrueEvaluator(DriverContext context) implements ExpressionEvaluator { @@ -227,6 +228,7 @@ public String toString() { } }; } + public static final ExpressionEvaluator.Factory CONSTANT_TRUE_FACTORY = new ConstantTrueEvaluator.Factory(); private record ConstantFalseEvaluator(DriverContext context) implements ExpressionEvaluator { @@ -263,5 +265,6 @@ public String toString() { } }; } + public static final ExpressionEvaluator.Factory CONSTANT_FALSE_FACTORY = new ConstantFalseEvaluator.Factory(); } From 66ba735c096b97c6290689d8183f24d80c585d73 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Fri, 29 Aug 2025 15:11:49 +0200 Subject: [PATCH 37/41] Extending test case to cover multirow null --- .../src/main/resources/string.csv-spec | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index a3976afeb09bc..d2fbc849318de 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -2072,6 +2072,32 @@ a:keyword | b:keyword | n:null | aa:boolean | bb:boolean | ab:boolean | ba:boo a | [a, b, c] | null | true | true | false | true | false | true | true ; +mvContainsCombinations_multirow +required_capability: fn_mv_contains + +ROW row_number = [1,2,3,4,5], element = "e", n = null, setA = ["b","d"], setB = ["a", "c", "e"] +| MV_EXPAND row_number +| EVAL superset = CASE( + row_number == 1, ["a","e"], + row_number == 2, ["b","d"], + row_number == 3, null, + row_number == 4, ["a","e","c","b","d"], + row_number == 5, ["a","d","c","b","e"], + null) +| EVAL contains_element = mv_contains(superset, element), + contains_null = mv_contains(superset, n), + contains_setA = mv_contains(superset, setA), + contains_setB = mv_contains(superset, setB) +; + +row_number:INTEGER | element:keyword | n:null | setA:keyword | setB:keyword | superset:keyword |contains_element:boolean | contains_null:boolean | contains_setA:boolean | contains_setB:boolean +1 | "e" | null | ["b","d"] | ["a", "c", "e"] | ["a","e"] | true | true | false | false +2 | "e" | null | ["b","d"] | ["a", "c", "e"] | ["b","d"] | false | true | true | false +3 | "e" | null | ["b","d"] | ["a", "c", "e"] | null | false | true | false | false +4 | "e" | null | ["b","d"] | ["a", "c", "e"] | ["a","e","c","b","d"] | true | true | true | true +5 | "e" | null | ["b","d"] | ["a", "c", "e"] | ["a","d","c","b","e"] | true | true | true | true +; + mvContains_where required_capability: fn_mv_contains // tag::mv_contains_where[] From 7492fad13b9b0d9f7dc3f4442817038ce836c7fb Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Fri, 29 Aug 2025 15:20:22 +0200 Subject: [PATCH 38/41] Remove unused `process` method from `MvContains` class. --- .../expression/function/scalar/multivalue/MvContains.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 8f1be8a924e34..dfdec763e330a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -206,14 +206,6 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { }; } - static void process(BooleanBlock.Builder builder, int position, Block subset) { - if (subset.isNull(position) == false) { - appendTo(builder, false); - return; - } - appendTo(builder, true); - } - // @Evaluator(extraName = "Int") see end of file. static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt)); From 23d92f0823b51d704875de9208bda532dc0335b7 Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Fri, 29 Aug 2025 15:47:29 +0200 Subject: [PATCH 39/41] Documenting changes done to copied test method due to inflexibility / visibility of methods / constructors. --- .../function/scalar/multivalue/MvContainsTests.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index cce33c1e76292..445480b986788 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -286,7 +286,11 @@ private static void bytesRefs(List suppliers) { })); } - // Adjusted from static method anyNullIsNull in {@code AbstractScalarFunctionTestCase#} + // Adjusted from static method anyNullIsNull in {@code AbstractFunctionTestCase#} + // - changed logic to expect a Boolean as an outcome and alternative evaluators (IsNullEvaluator, ConstantTrue) + // - constructor TestCase that's used has default access which we can't access, using public constructor variant as a replacement + // - Added prefix to generated tests for my sanity. + // - changed construction of new lists by copying them and updating the entries where necessary instead of regenerating protected static List anyNullIsNull( List testCaseSuppliers, ExpectedType expectedType, From d33e3332bafacbfa114a86b587f57e9a6ab9b6ad Mon Sep 17 00:00:00 2001 From: mjmbischoff Date: Fri, 29 Aug 2025 16:27:26 +0200 Subject: [PATCH 40/41] Always true. --- .../function/scalar/multivalue/MvContainsTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index 445480b986788..b4f2bc6318193 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -364,9 +364,9 @@ private static String toSpaceSeparatedString(ArrayList typesWithNull) return typesWithNull.stream().map(Objects::toString).collect(Collectors.joining(" ")); } - // We always return a boolean. + // When all arguments are null: the 2nd arg (subset) will be `null` and the 1st is invariant (null,null) => true. @Override protected Matcher allNullsMatcher() { - return anyOf(equalTo(false), equalTo(true)); + return equalTo(true); } } From a31922ba269a1e5ad54dfe43148bad4208c9f61a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 29 Aug 2025 14:37:30 +0000 Subject: [PATCH 41/41] [CI] Auto commit changes from spotless --- .../expression/function/scalar/multivalue/MvContainsTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java index b4f2bc6318193..57da6d5c2fe9a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsTests.java @@ -35,7 +35,6 @@ import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TypedData.MULTI_ROW_NULL; import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TypedData.NULL; -import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; public class MvContainsTests extends AbstractScalarFunctionTestCase {