From 3079bad3bb4951e4158ccfcdd1f54432004f1c1b Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 14 Aug 2023 18:34:20 +0200 Subject: [PATCH 01/10] Add support for TEXT operator for binary comparison and order --- .../compute/operator/EvalBenchmark.java | 2 +- .../resources/rest-api-spec/test/80_text.yml | 163 ++++++++++++++++++ .../xpack/esql/analysis/Verifier.java | 7 +- .../xpack/esql/expression/Order.java | 40 +++++ .../predicate/operator/comparison/Equals.java | 47 ++++- .../operator/comparison/GreaterThan.java | 48 +++++- .../comparison/GreaterThanOrEqual.java | 48 +++++- .../operator/comparison/LessThan.java | 52 +++++- .../operator/comparison/LessThanOrEqual.java | 48 +++++- .../operator/comparison/NotEquals.java | 43 ++++- .../operator/comparison/NullEquals.java | 50 ++++++ .../xpack/esql/io/stream/PlanNamedTypes.java | 16 +- .../optimizer/LocalPhysicalPlanOptimizer.java | 24 ++- .../esql/optimizer/LogicalPlanOptimizer.java | 124 ++++++++++++- .../xpack/esql/parser/ExpressionBuilder.java | 12 +- .../xpack/esql/planner/ComparisonMapper.java | 12 +- .../xpack/esql/planner/InMapper.java | 2 +- .../AbstractBinaryComparisonTestCase.java | 4 +- .../operator/comparison/EqualsTests.java | 1 - .../comparison/GreaterThanOrEqualTests.java | 1 - .../operator/comparison/GreaterThanTests.java | 1 - .../comparison/LessThanOrEqualTests.java | 1 - .../operator/comparison/LessThanTests.java | 1 - .../operator/comparison/NotEqualsTests.java | 1 - .../esql/io/stream/PlanNamedTypesTests.java | 23 ++- .../optimizer/LogicalPlanOptimizerTests.java | 31 ++-- .../optimizer/PhysicalPlanOptimizerTests.java | 10 +- .../xpack/esql/parser/ExpressionTests.java | 8 +- .../esql/parser/StatementParserTests.java | 10 +- .../xpack/esql/planner/EvalMapperTests.java | 10 +- 30 files changed, 755 insertions(+), 85 deletions(-) create mode 100644 x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java index 4c1e9087ded76..389123633067f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java @@ -19,13 +19,13 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.planner.EvalMapper; import org.elasticsearch.xpack.esql.planner.Layout; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.EsField; diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml new file mode 100644 index 0000000000000..5d18f7454e1e4 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml @@ -0,0 +1,163 @@ +--- +setup: + + - do: + indices.create: + index: test + body: + mappings: + properties: + "emp_no": + type: long + name: + type: keyword + job: + type: text + fields: + raw: + type: keyword + tag: + type: text + + - do: + bulk: + index: test + refresh: true + body: + - { "index": { } } + - { "emp_no": 10, "name": "Jenny", "job": "IT Director", "tag": "foo bar" } + - { "index": { } } + - { "emp_no": 20, "name": "John", "job": "Payroll Specialist", "tag": "baz" } +--- +"filter by text": + - do: + esql.query: + body: + query: 'from test | where tag == "baz" | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 1 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + +--- +"eval and filter text": + - do: + esql.query: + body: + query: 'from test | eval x = tag | where x == "baz" | keep emp_no, name, job, x' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "x" } + - match: { columns.3.type: "text" } + + - length: { values: 1 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + +--- +"filter on text multi-field": + - do: + esql.query: + body: + query: 'from test | where job == "IT Director" | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 1 } + - match: { values.0: [ 10, "Jenny", "IT Director", "foo bar"] } + + +--- +"sort by text": + - do: + esql.query: + body: + query: 'from test | sort tag | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 2 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + - match: { values.1: [ 10, "Jenny", "IT Director", "foo bar"] } + + +--- +"sort by text multi-field": + - do: + esql.query: + body: + query: 'from test | sort job | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 2 } + - match: { values.0: [ 10, "Jenny", "IT Director", "foo bar"] } + - match: { values.1: [ 20, "John", "Payroll Specialist", "baz"] } + +--- +"sort by text multi-field desc": + - do: + esql.query: + body: + query: 'from test | sort job desc | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 2 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + - match: { values.1: [ 10, "Jenny", "IT Director", "foo bar"] } + + +--- +"text in functions": + - do: + esql.query: + body: + query: 'from test | sort name | eval description = concat(name, " - ", job) | keep description' + + - match: { columns.0.name: "description" } + - match: { columns.0.type: "keyword" } + + - length: { values: 2 } + - match: { values.0: [ "Jenny - IT Director"] } + - match: { values.1: [ "John - Payroll Specialist"] } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 40e16e33205e9..19887767887fd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Grok; @@ -28,8 +30,6 @@ import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.ql.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.Limit; @@ -258,6 +258,9 @@ public static Failure validateBinaryComparison(BinaryComparison bc) { if (false == r.resolved()) { return fail(bc, r.message()); } + if (DataTypes.isString(bc.left().dataType()) && DataTypes.isString(bc.right().dataType())) { + return null; + } if (bc.left().dataType() != bc.right().dataType()) { return fail( bc, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java new file mode 100644 index 0000000000000..95852a00ce2bb --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.List; + +public class Order extends org.elasticsearch.xpack.ql.expression.Order { + public Order(Source source, Expression child, OrderDirection direction, NullsPosition nulls) { + super(source, child, direction, nulls); + } + + @Override + protected TypeResolution resolveType() { + if (DataTypes.isString(child().dataType())) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveType(); + } + + @Override + public Order replaceChildren(List newChildren) { + return new Order(source(), newChildren.get(0), direction(), nullsPosition()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Order::new, child(), direction(), nullsPosition()); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 7793dc0f8e167..1bda71807f7cf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -8,8 +8,53 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; + +public class Equals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals { + public Equals(Source source, Expression left, Expression right) { + super(source, left, right); + } + + public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveInputType(e, paramOrdinal); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Equals::new, left(), right(), zoneId()); + } + + @Override + protected Equals replaceChildren(Expression newLeft, Expression newRight) { + return new Equals(source(), newLeft, newRight, zoneId()); + } + + @Override + public Equals swapLeftAndRight() { + return new Equals(source(), right(), left(), zoneId()); + } + + @Override + public BinaryComparison negate() { + return new NotEquals(source(), left(), right(), zoneId()); + } -public class Equals { @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs == rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java index 84be8eb00c99e..2ab72ef330916 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java @@ -8,8 +8,54 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; + +public class GreaterThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan { + public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveInputType(e, paramOrdinal); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, GreaterThan::new, left(), right(), zoneId()); + } + + @Override + protected GreaterThan replaceChildren(Expression newLeft, Expression newRight) { + return new GreaterThan(source(), newLeft, newRight, zoneId()); + } + + @Override + public LessThan swapLeftAndRight() { + return new LessThan(source(), right(), left(), zoneId()); + } + + @Override + public LessThanOrEqual negate() { + return new LessThanOrEqual(source(), left(), right(), zoneId()); + } + + @Override + public BinaryComparison reverse() { + return new LessThan(source(), left(), right(), zoneId()); + } -public class GreaterThan { @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs > rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java index 86ce56c7e3bc5..9f5bd1b87e18a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java @@ -8,8 +8,54 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; -public class GreaterThanOrEqual { +import java.time.ZoneId; + +public class GreaterThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual { + + public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveInputType(e, paramOrdinal); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, GreaterThanOrEqual::new, left(), right(), zoneId()); + } + + @Override + protected GreaterThanOrEqual replaceChildren(Expression newLeft, Expression newRight) { + return new GreaterThanOrEqual(source(), newLeft, newRight, zoneId()); + } + + @Override + public LessThanOrEqual swapLeftAndRight() { + return new LessThanOrEqual(source(), right(), left(), zoneId()); + } + + @Override + public LessThan negate() { + return new LessThan(source(), left(), right(), zoneId()); + } + + @Override + public BinaryComparison reverse() { + return new LessThanOrEqual(source(), left(), right(), zoneId()); + } @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java index da9540b1f0442..5166271b9bb4d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java @@ -8,8 +8,58 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; + +public class LessThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan { + + public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveInputType(e, paramOrdinal); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, LessThan::new, left(), right(), zoneId()); + } + + @Override + protected org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan replaceChildren( + Expression newLeft, + Expression newRight + ) { + return new LessThan(source(), newLeft, newRight, zoneId()); + } + + @Override + public GreaterThan swapLeftAndRight() { + return new GreaterThan(source(), right(), left(), zoneId()); + } + + @Override + public GreaterThanOrEqual negate() { + return new GreaterThanOrEqual(source(), left(), right(), zoneId()); + } + + @Override + public BinaryComparison reverse() { + return new GreaterThan(source(), left(), right(), zoneId()); + } -public class LessThan { @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs < rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index af06e80f922c2..225a74d4ad4d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -8,8 +8,54 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; + +public class LessThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual { + public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveInputType(e, paramOrdinal); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, LessThanOrEqual::new, left(), right(), zoneId()); + } + + @Override + protected LessThanOrEqual replaceChildren(Expression newLeft, Expression newRight) { + return new LessThanOrEqual(source(), newLeft, newRight, zoneId()); + } + + @Override + public GreaterThanOrEqual swapLeftAndRight() { + return new GreaterThanOrEqual(source(), right(), left(), zoneId()); + } + + @Override + public GreaterThan negate() { + return new GreaterThan(source(), left(), right(), zoneId()); + } + + @Override + public BinaryComparison reverse() { + return super.reverse(); + } -public class LessThanOrEqual { @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs <= rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java index 9d31e661c93d2..8c6e102ff8f17 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java @@ -8,8 +8,49 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; + +public class NotEquals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals { + public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveInputType(e, paramOrdinal); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); + } + + @Override + protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { + return new NotEquals(source(), newLeft, newRight, zoneId()); + } + + @Override + public NotEquals swapLeftAndRight() { + return new NotEquals(source(), right(), left(), zoneId()); + } + + @Override + public BinaryComparison negate() { + return new Equals(source(), left(), right(), zoneId()); + } -public class NotEquals { @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs != rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java new file mode 100644 index 0000000000000..9e2b45967883c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; + +public class NullEquals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals { + public NullEquals(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { + return TypeResolution.TYPE_RESOLVED; + } + return super.resolveInputType(e, paramOrdinal); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, NullEquals::new, left(), right(), zoneId()); + } + + @Override + protected org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals replaceChildren( + Expression newLeft, + Expression newRight + ) { + return new NullEquals(source(), newLeft, newRight, zoneId()); + } + + @Override + public org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals swapLeftAndRight() { + return new NullEquals(source(), right(), left(), zoneId()); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 52748ad987992..142bd803d7ec4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -81,7 +81,14 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Dissect.Parser; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -136,13 +143,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; @@ -1366,7 +1366,7 @@ static void writeLiteral(PlanStreamOutput out, Literal literal) throws IOExcepti } static Order readOrder(PlanStreamInput in) throws IOException { - return new Order( + return new org.elasticsearch.xpack.esql.expression.Order( Source.EMPTY, in.readNamed(Expression.class), in.readEnum(Order.OrderDirection.class), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java index 74e1a9e0cc180..80e476e2fa50b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java @@ -9,7 +9,9 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules.OptimizerRule; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; @@ -37,8 +39,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike; import org.elasticsearch.xpack.ql.planner.ExpressionTranslator; @@ -232,7 +232,7 @@ private static boolean canPushToSource(Expression exp) { } private static boolean isAttributePushable(Expression expression, ScalarFunction operation) { - if (expression instanceof FieldAttribute) { + if (expression instanceof FieldAttribute f && f.getExactInfo().hasExact()) { return true; } if (expression instanceof MetadataAttribute ma && ma.searchable()) { @@ -283,13 +283,22 @@ protected PhysicalPlan rule(TopNExec topNExec) { private boolean canPushDownOrders(List orders) { // allow only FieldAttributes (no expressions) for sorting - return false == Expressions.match(orders, s -> ((Order) s).child() instanceof FieldAttribute == false); + for (Order order : orders) { + if (order.child() instanceof FieldAttribute fa) { + if (fa.getExactInfo().hasExact() == false) { + return false; + } + } else { + return false; + } + } + return true; } private List buildFieldSorts(List orders) { List sorts = new ArrayList<>(orders.size()); for (Order o : orders) { - sorts.add(new EsQueryExec.FieldSort(((FieldAttribute) o.child()), o.direction(), o.nullsPosition())); + sorts.add(new EsQueryExec.FieldSort(((FieldAttribute) o.child()).exactAttribute(), o.direction(), o.nullsPosition())); } return sorts; } @@ -299,7 +308,10 @@ private static final class EsqlTranslatorHandler extends QlTranslatorHandler { @Override public Query wrapFunctionQuery(ScalarFunction sf, Expression field, Supplier querySupplier) { if (field instanceof FieldAttribute fa) { - return ExpressionTranslator.wrapIfNested(new SingleValueQuery(querySupplier.get(), fa.name()), field); + return ExpressionTranslator.wrapIfNested( + new SingleValueQuery(querySupplier.get(), fa.exactAttribute().name()), + ((FieldAttribute) field).exactAttribute() + ); } if (field instanceof MetadataAttribute) { return querySupplier.get(); // MetadataAttributes are always single valued diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index c33a6c3f34432..768f51dab173a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -35,6 +35,10 @@ import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.ql.expression.predicate.Predicates; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; +import org.elasticsearch.xpack.ql.expression.predicate.regex.StringPattern; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BinaryComparisonSimplification; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanFunctionEqualsElimination; @@ -51,23 +55,29 @@ import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.ql.rule.RuleExecutor; +import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.CollectionUtils; import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.ql.expression.Expressions.asAttributes; +import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.combineOr; +import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.splitOr; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.FoldNull; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateNullable; -import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceRegexMatch; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.TransformDirection; public class LogicalPlanOptimizer extends RuleExecutor { @@ -666,9 +676,86 @@ private static Project pushDownPastProject(UnaryPlan parent) { } } - static class CombineDisjunctionsToIn extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineDisjunctionsToIn { + /** + * Combine disjunctions on the same field into an In expression. + * This rule looks for both simple equalities: + * 1. a == 1 OR a == 2 becomes a IN (1, 2) + * and combinations of In + * 2. a == 1 OR a IN (2) becomes a IN (1, 2) + * 3. a IN (1) OR a IN (2) becomes a IN (1, 2) + * + * This rule does NOT check for type compatibility as that phase has been + * already be verified in the analyzer. + */ + public static class CombineDisjunctionsToIn extends OptimizerRules.OptimizerExpressionRule { + public CombineDisjunctionsToIn() { + super(TransformDirection.UP); + } + @Override - protected In createIn(Expression key, List values, ZoneId zoneId) { + protected Expression rule(Or or) { + Expression e = or; + // look only at equals and In + List exps = splitOr(e); + + Map> found = new LinkedHashMap<>(); + ZoneId zoneId = null; + List ors = new LinkedList<>(); + + for (Expression exp : exps) { + if (exp instanceof Equals eq) { + // consider only equals against foldables + if (eq.right().foldable()) { + found.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right()); + } else { + ors.add(exp); + } + if (zoneId == null) { + zoneId = eq.zoneId(); + } + } else if (exp instanceof org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In in) { + found.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(in.list()); + if (zoneId == null) { + zoneId = in.zoneId(); + } + } else { + ors.add(exp); + } + } + + if (found.isEmpty() == false) { + // combine equals alongside the existing ors + final ZoneId finalZoneId = zoneId; + found.forEach((k, v) -> { + ors.add( + v.size() == 1 + ? new org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals( + k.source(), + k, + v.iterator().next(), + finalZoneId + ) + : createIn(k, new ArrayList<>(v), finalZoneId) + ); + }); + + Expression combineOr = combineOr(ors); + // check the result semantically since the result might different in order + // but be actually the same which can trigger a loop + // e.g. a == 1 OR a == 2 OR null --> null OR a in (1,2) --> literalsOnTheRight --> cycle + if (e.semanticEquals(combineOr) == false) { + e = combineOr; + } + } + + return e; + } + + protected org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In createIn( + Expression key, + List values, + ZoneId zoneId + ) { return new In(key.source(), key, values); } } @@ -684,4 +771,35 @@ protected LogicalPlan rule(Limit plan) { return p; } } + + public static class ReplaceRegexMatch extends OptimizerRules.OptimizerExpressionRule> { + + public ReplaceRegexMatch() { + super(TransformDirection.DOWN); + } + + @Override + protected Expression rule(RegexMatch regexMatch) { + Expression e = regexMatch; + StringPattern pattern = regexMatch.pattern(); + if (pattern.matchesAll()) { + e = new IsNotNull(e.source(), regexMatch.field()); + } else { + String match = pattern.exactMatch(); + if (match != null) { + Literal literal = new Literal(regexMatch.source(), match, DataTypes.KEYWORD); + e = regexToEquals(regexMatch, literal); + } + } + return e; + } + + protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { + return new org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals( + regexMatch.source(), + regexMatch.field(), + literal + ); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index f8e0ee5dac439..bdc6ed7fedbf7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -12,14 +12,19 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.TerminalNode; import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.esql.expression.Order; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.NamedExpression; -import org.elasticsearch.xpack.ql.expression.Order; import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.ql.expression.UnresolvedStar; import org.elasticsearch.xpack.ql.expression.function.FunctionResolutionStrategy; @@ -33,11 +38,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ComparisonMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ComparisonMapper.java index eb2aa832d3ed3..08bcbc783e89d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ComparisonMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ComparisonMapper.java @@ -10,15 +10,15 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import org.elasticsearch.xpack.ql.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/InMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/InMapper.java index 03a92ddeb7bed..7b47da07530fd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/InMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/InMapper.java @@ -15,8 +15,8 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import java.util.ArrayList; import java.util.BitSet; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/AbstractBinaryComparisonTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/AbstractBinaryComparisonTestCase.java index e21b9947271fa..7adb6774e4251 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/AbstractBinaryComparisonTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/AbstractBinaryComparisonTestCase.java @@ -94,7 +94,9 @@ protected final void validateType(BinaryOperator op, DataType lhsTyp ); return; } - if (lhsType == rhsType || lhsType.isNumeric() && rhsType.isNumeric()) { + if (lhsType == rhsType + || lhsType.isNumeric() && rhsType.isNumeric() + || DataTypes.isString(lhsType) && DataTypes.isString(rhsType)) { assertThat(op.toString(), f, nullValue()); return; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 60dcccc0f4a2d..891c8d8556479 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index c108f965f6e68..eb922a475990c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index 561cde534e47e..5e6f82e96c41f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index bec73c260776d..bfe193085136f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index aa80d08c56605..237880bb9a821 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index cc25e4169a441..8e57808b32e39 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java index 99114fc48311e..8340847dd6712 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java @@ -31,6 +31,13 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Round; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.DissectExec; @@ -69,13 +76,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -354,11 +354,16 @@ public void testLiteralSimple() throws IOException { } public void testOrderSimple() throws IOException { - var orig = new Order(Source.EMPTY, field("val", DataTypes.INTEGER), Order.OrderDirection.ASC, Order.NullsPosition.FIRST); + var orig = new org.elasticsearch.xpack.esql.expression.Order( + Source.EMPTY, + field("val", DataTypes.INTEGER), + Order.OrderDirection.ASC, + Order.NullsPosition.FIRST + ); BytesStreamOutput bso = new BytesStreamOutput(); PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry); PlanNamedTypes.writeOrder(out, orig); - var deser = PlanNamedTypes.readOrder(planStreamInput(bso)); + var deser = (org.elasticsearch.xpack.esql.expression.Order) PlanNamedTypes.readOrder(planStreamInput(bso)); EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index e69a46e5cc5d9..4810dcdedec00 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -27,7 +27,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Round; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -52,10 +56,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike; import org.elasticsearch.xpack.ql.index.EsIndex; @@ -85,9 +85,6 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptySource; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource; -import static org.elasticsearch.xpack.ql.TestUtils.greaterThanOf; -import static org.elasticsearch.xpack.ql.TestUtils.greaterThanOrEqualOf; -import static org.elasticsearch.xpack.ql.TestUtils.lessThanOf; import static org.elasticsearch.xpack.ql.TestUtils.relation; import static org.elasticsearch.xpack.ql.TestUtils.rlike; import static org.elasticsearch.xpack.ql.TestUtils.wildcardLike; @@ -285,6 +282,18 @@ public void testMultipleCombineLimits() { assertEquals(new Limit(EMPTY, L(minimum), relation), new LogicalPlanOptimizer().optimize(plan)); } + public static GreaterThan greaterThanOf(Expression left, Expression right) { + return new GreaterThan(EMPTY, left, right, randomZone()); + } + + public static LessThan lessThanOf(Expression left, Expression right) { + return new LessThan(EMPTY, left, right, randomZone()); + } + + public static GreaterThanOrEqual greaterThanOrEqualOf(Expression left, Expression right) { + return new GreaterThanOrEqual(EMPTY, left, right, randomZone()); + } + public void testCombineFilters() { EsRelation relation = relation(); GreaterThan conditionA = greaterThanOf(getFieldAttribute("a"), ONE); @@ -908,19 +917,19 @@ public void testPruneRedundantSortClauses() { assertThat( topN.order(), contains( - new Order( + new org.elasticsearch.xpack.esql.expression.Order( EMPTY, new ReferenceAttribute(EMPTY, "e", INTEGER, null, Nullability.TRUE, null, false), Order.OrderDirection.ASC, Order.NullsPosition.LAST ), - new Order( + new org.elasticsearch.xpack.esql.expression.Order( EMPTY, new FieldAttribute(EMPTY, "emp_no", mapping.get("emp_no")), Order.OrderDirection.ASC, Order.NullsPosition.LAST ), - new Order( + new org.elasticsearch.xpack.esql.expression.Order( EMPTY, new FieldAttribute(EMPTY, "salary", mapping.get("salary")), Order.OrderDirection.DESC, @@ -944,7 +953,7 @@ public void testPruneRedundantSortClausesUsingAlias() { assertThat( topN.order(), contains( - new Order( + new org.elasticsearch.xpack.esql.expression.Order( EMPTY, new FieldAttribute(EMPTY, "emp_no", mapping.get("emp_no")), Order.OrderDirection.ASC, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 709c62c0b0fb4..405fa8de7100e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -29,6 +29,11 @@ import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolution; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Round; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; @@ -66,11 +71,6 @@ import org.elasticsearch.xpack.ql.expression.Order; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.type.DataTypes; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java index 3469cc66f21aa..cf14785afac05 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java @@ -8,6 +8,10 @@ package org.elasticsearch.xpack.esql.parser; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.ql.expression.Alias; @@ -24,10 +28,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.Project; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 25658905297d5..2129491ad5662 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -9,6 +9,11 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; @@ -29,11 +34,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java index 2ac38491f1666..797cf1c6bfc51 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java @@ -22,6 +22,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; @@ -34,11 +39,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Div; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; From 76d9c7b8bbfb48a1a25cbc72ca4f436925150d11 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 16 Aug 2023 11:17:52 +0200 Subject: [PATCH 02/10] Add support TEXT fields in LIKE and RLIKE --- .../resources/rest-api-spec/test/80_text.yml | 76 +++++++++++++++++++ .../predicate/operator/regex/RLike.java | 41 ++++++++++ .../operator/regex/WildcardLike.java | 37 +++++++++ .../xpack/esql/io/stream/PlanNamedTypes.java | 6 +- .../xpack/esql/parser/ExpressionBuilder.java | 6 +- .../esql/parser/StatementParserTests.java | 4 +- 6 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/RLike.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml index 5d18f7454e1e4..469dee6c32316 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml @@ -47,6 +47,44 @@ setup: - length: { values: 1 } - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } +--- +"like by text": + - do: + esql.query: + body: + query: 'from test | where tag LIKE "*az" | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 1 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + +--- +"rlike by text": + - do: + esql.query: + body: + query: 'from test | where tag RLIKE ".*az" | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 1 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + --- "eval and filter text": - do: @@ -85,6 +123,44 @@ setup: - length: { values: 1 } - match: { values.0: [ 10, "Jenny", "IT Director", "foo bar"] } +--- +"like by multi-field text": + - do: + esql.query: + body: + query: 'from test | where job LIKE "*Specialist" | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 1 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + +--- +"rlike by multi-field text": + - do: + esql.query: + body: + query: 'from test | where job RLIKE ".*Specialist" | keep emp_no, name, job, tag' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "name" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "job" } + - match: { columns.2.type: "text" } + - match: { columns.3.name: "tag" } + - match: { columns.3.type: "text" } + + - length: { values: 1 } + - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] } + --- "sort by text": diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/RLike.java new file mode 100644 index 0000000000000..ae5358a1e4f3b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/RLike.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.regex; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; + +public class RLike extends org.elasticsearch.xpack.ql.expression.predicate.regex.RLike { + public RLike(Source source, Expression value, RLikePattern pattern) { + super(source, value, pattern); + } + + public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) { + super(source, field, rLikePattern, caseInsensitive); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, RLike::new, field(), pattern(), caseInsensitive()); + } + + @Override + protected RLike replaceChild(Expression newChild) { + return new RLike(source(), newChild, pattern(), caseInsensitive()); + } + + @Override + protected TypeResolution resolveType() { + return isString(field(), sourceText(), DEFAULT); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java new file mode 100644 index 0000000000000..8b52ced73e238 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.regex; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardPattern; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; + +public class WildcardLike extends org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike { + public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) { + super(source, left, pattern, caseInsensitive); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, WildcardLike::new, field(), pattern(), caseInsensitive()); + } + + @Override + protected WildcardLike replaceChild(Expression newLeft) { + return new WildcardLike(source(), newLeft, pattern(), caseInsensitive()); + } + + @Override + protected TypeResolution resolveType() { + return isString(field(), sourceText(), DEFAULT); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 142bd803d7ec4..c632b5b622fdd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -89,6 +89,8 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NullEquals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.regex.RLike; +import org.elasticsearch.xpack.esql.expression.predicate.operator.regex.WildcardLike; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Dissect.Parser; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -143,10 +145,8 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; -import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; -import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; @@ -971,7 +971,7 @@ static void writeInComparison(PlanStreamOutput out, In in) throws IOException { // -- RegexMatch static WildcardLike readWildcardLike(PlanStreamInput in, String name) throws IOException { - return new WildcardLike(Source.EMPTY, in.readExpression(), new WildcardPattern(in.readString())); + return new WildcardLike(Source.EMPTY, in.readExpression(), new WildcardPattern(in.readString()), false); } static void writeWildcardLike(PlanStreamOutput out, WildcardLike like) throws IOException { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index bdc6ed7fedbf7..1d531ec7f4369 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -19,6 +19,8 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.regex.RLike; +import org.elasticsearch.xpack.esql.expression.predicate.operator.regex.WildcardLike; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Alias; @@ -38,10 +40,8 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; -import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; -import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -345,7 +345,7 @@ public Expression visitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpress Expression left = expression(ctx.valueExpression()); Literal pattern = visitString(ctx.pattern); RegexMatch result = switch (type) { - case EsqlBaseParser.LIKE -> new WildcardLike(source, left, new WildcardPattern(pattern.fold().toString())); + case EsqlBaseParser.LIKE -> new WildcardLike(source, left, new WildcardPattern(pattern.fold().toString()), false); case EsqlBaseParser.RLIKE -> new RLike(source, left, new RLikePattern(pattern.fold().toString())); default -> throw new ParsingException("Invalid predicate type for [{}]", source.text()); }; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 2129491ad5662..ae21cd393de8c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.regex.RLike; +import org.elasticsearch.xpack.esql.expression.predicate.operator.regex.WildcardLike; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; @@ -34,8 +36,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; -import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.Limit; From 515dbe669bfb8f620c2dedd8ad5706958bb450f9 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 16 Aug 2023 11:46:08 +0200 Subject: [PATCH 03/10] Add tests --- .../optimizer/PhysicalPlanOptimizerTests.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 405fa8de7100e..49e1fa1f15c16 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -1653,6 +1653,41 @@ public void testDontPushDownMetadataVersionAndId() { } } + public void testNoTextFilterPushDown() { + var plan = physicalPlan(""" + from test + | where gender == "M" + """); + + var optimized = optimizedPlan(plan); + var limit = as(optimized, LimitExec.class); + var exchange = asRemoteExchange(limit.child()); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + var limit2 = as(extract.child(), LimitExec.class); + var filter = as(limit2.child(), FilterExec.class); + var extract2 = as(filter.child(), FieldExtractExec.class); + var source = source(extract2.child()); + assertNull(source.query()); + } + + public void testNoTextSortPushDown() { + var plan = physicalPlan(""" + from test + | sort gender + """); + + var optimized = optimizedPlan(plan); + var topN = as(optimized, TopNExec.class); + var exchange = as(topN.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + var topN2 = as(extract.child(), TopNExec.class); + var extract2 = as(topN2.child(), FieldExtractExec.class); + var source = source(extract2.child()); + assertNull(source.sorts()); + } + private static EsQueryExec source(PhysicalPlan plan) { if (plan instanceof ExchangeExec exchange) { plan = exchange.child(); From c7398868b6386cbccc7908dde1aae2c0ebac6283 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 17 Aug 2023 09:39:55 +0200 Subject: [PATCH 04/10] Implement review suggestions --- .../operator/comparison/LessThanOrEqual.java | 2 +- .../predicate/operator/regex/WildcardLike.java | 8 ++++---- .../xpack/esql/io/stream/PlanNamedTypes.java | 2 +- .../xpack/esql/parser/ExpressionBuilder.java | 2 +- .../xpack/esql/io/stream/PlanNamedTypesTests.java | 11 +++-------- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index 225a74d4ad4d6..546f08b2621e1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -53,7 +53,7 @@ public GreaterThan negate() { @Override public BinaryComparison reverse() { - return super.reverse(); + return new GreaterThanOrEqual(source(), left(), right(), zoneId()); } @Evaluator(extraName = "Ints") diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java index 8b52ced73e238..398cda4c3c2b1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/regex/WildcardLike.java @@ -16,18 +16,18 @@ import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; public class WildcardLike extends org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardLike { - public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) { - super(source, left, pattern, caseInsensitive); + public WildcardLike(Source source, Expression left, WildcardPattern pattern) { + super(source, left, pattern, false); } @Override protected NodeInfo info() { - return NodeInfo.create(this, WildcardLike::new, field(), pattern(), caseInsensitive()); + return NodeInfo.create(this, WildcardLike::new, field(), pattern()); } @Override protected WildcardLike replaceChild(Expression newLeft) { - return new WildcardLike(source(), newLeft, pattern(), caseInsensitive()); + return new WildcardLike(source(), newLeft, pattern()); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index c632b5b622fdd..f645a163e79b8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -971,7 +971,7 @@ static void writeInComparison(PlanStreamOutput out, In in) throws IOException { // -- RegexMatch static WildcardLike readWildcardLike(PlanStreamInput in, String name) throws IOException { - return new WildcardLike(Source.EMPTY, in.readExpression(), new WildcardPattern(in.readString()), false); + return new WildcardLike(Source.EMPTY, in.readExpression(), new WildcardPattern(in.readString())); } static void writeWildcardLike(PlanStreamOutput out, WildcardLike like) throws IOException { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 1d531ec7f4369..4457d364891d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -345,7 +345,7 @@ public Expression visitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpress Expression left = expression(ctx.valueExpression()); Literal pattern = visitString(ctx.pattern); RegexMatch result = switch (type) { - case EsqlBaseParser.LIKE -> new WildcardLike(source, left, new WildcardPattern(pattern.fold().toString()), false); + case EsqlBaseParser.LIKE -> new WildcardLike(source, left, new WildcardPattern(pattern.fold().toString())); case EsqlBaseParser.RLIKE -> new RLike(source, left, new RLikePattern(pattern.fold().toString())); default -> throw new ParsingException("Invalid predicate type for [{}]", source.text()); }; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java index 8340847dd6712..ff605defe72a1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.SerializationTestUtils; +import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.Avg; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; @@ -67,7 +68,6 @@ import org.elasticsearch.xpack.ql.expression.NameId; import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.Nullability; -import org.elasticsearch.xpack.ql.expression.Order; import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.ArithmeticOperation; @@ -354,16 +354,11 @@ public void testLiteralSimple() throws IOException { } public void testOrderSimple() throws IOException { - var orig = new org.elasticsearch.xpack.esql.expression.Order( - Source.EMPTY, - field("val", DataTypes.INTEGER), - Order.OrderDirection.ASC, - Order.NullsPosition.FIRST - ); + var orig = new Order(Source.EMPTY, field("val", DataTypes.INTEGER), Order.OrderDirection.ASC, Order.NullsPosition.FIRST); BytesStreamOutput bso = new BytesStreamOutput(); PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry); PlanNamedTypes.writeOrder(out, orig); - var deser = (org.elasticsearch.xpack.esql.expression.Order) PlanNamedTypes.readOrder(planStreamInput(bso)); + var deser = (Order) PlanNamedTypes.readOrder(planStreamInput(bso)); EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); } From b80187c33bd681e416b4fb76475fea4b6fa08c9b Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 23 Aug 2023 14:18:28 +0200 Subject: [PATCH 05/10] Use exact fields and fix stats validation --- .../resources/rest-api-spec/test/80_text.yml | 79 ++++++++-- .../src/main/resources/mapping-basic.json | 8 + .../expression/function/aggregate/Count.java | 5 + .../function/aggregate/CountDistinct.java | 2 +- .../optimizer/LocalPhysicalPlanOptimizer.java | 20 +-- .../esql/optimizer/LogicalPlanOptimizer.java | 141 ++++++------------ .../xpack/esql/analysis/AnalyzerTests.java | 18 ++- .../optimizer/LogicalPlanOptimizerTests.java | 37 +++++ .../optimizer/PhysicalPlanOptimizerTests.java | 65 +++++++- .../xpack/ql/optimizer/OptimizerRules.java | 6 +- 10 files changed, 245 insertions(+), 136 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml index 469dee6c32316..2e197cc364da7 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml @@ -40,7 +40,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -59,7 +59,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -78,7 +78,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -97,7 +97,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "x" } - match: { columns.3.type: "text" } @@ -116,7 +116,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -135,7 +135,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -154,7 +154,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -174,7 +174,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -195,7 +195,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -215,7 +215,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "text" } + - match: { columns.2.type: "keyword" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -237,3 +237,62 @@ setup: - length: { values: 2 } - match: { values.0: [ "Jenny - IT Director"] } - match: { values.1: [ "John - Payroll Specialist"] } + + +--- +"stats text with raw": + - do: + esql.query: + body: + query: 'from test | stats jobs = count(job) | keep jobs' + + - match: { columns.0.name: "jobs" } + - match: { columns.0.type: "long" } + + - length: { values: 1 } + - match: { values.0: [ 2 ] } + + +--- +"stats text no raw": + - do: + esql.query: + body: + query: 'from test | stats tags = count(tag) | keep tags' + + - match: { columns.0.name: "tags" } + - match: { columns.0.type: "long" } + + - length: { values: 1 } + - match: { values.0: [ 2 ] } + + +--- +"stats by text with raw": + - do: + esql.query: + body: + query: 'from test | stats names = count(name) by job | keep names' + + - match: { columns.0.name: "names" } + - match: { columns.0.type: "long" } + + - length: { values: 2 } + - match: { values.0: [ 1 ] } + - match: { values.0: [ 1 ] } + + +--- +"stats by text no raw": + - do: + esql.query: + body: + query: 'from test | stats names = count(name) by tag | keep names' + + - match: { columns.0.name: "names" } + - match: { columns.0.type: "long" } + + - length: { values: 2 } + - match: { values.0: [ 1 ] } + - match: { values.0: [ 1 ] } + diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json index 7edd242c50a7c..b650cb7e64564 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json @@ -20,6 +20,14 @@ }, "_meta_field": { "type" : "keyword" + }, + "job": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index fce52374bfab0..26adbb255b0c7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -57,4 +57,9 @@ public AggregatorFunctionSupplier supplier(BigArrays bigArrays, List in public Nullability nullable() { return Nullability.FALSE; } + + @Override + protected TypeResolution resolveType() { + return field().dataType() == DataTypes.TEXT ? TypeResolution.TYPE_RESOLVED : super.resolveType(); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 4e2dbc98bdafc..f17b9fa144124 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -58,7 +58,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = super.resolveType(); + TypeResolution resolution = field().dataType() == DataTypes.TEXT ? TypeResolution.TYPE_RESOLVED : super.resolveType(); if (resolution.unresolved() || precision == null) { return resolution; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java index 80e476e2fa50b..fa0c792f263d0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java @@ -282,23 +282,14 @@ protected PhysicalPlan rule(TopNExec topNExec) { } private boolean canPushDownOrders(List orders) { - // allow only FieldAttributes (no expressions) for sorting - for (Order order : orders) { - if (order.child() instanceof FieldAttribute fa) { - if (fa.getExactInfo().hasExact() == false) { - return false; - } - } else { - return false; - } - } - return true; + // allow only exact FieldAttributes (no expressions) for sorting + return orders.stream().allMatch(o -> o.child() instanceof FieldAttribute fa && fa.getExactInfo().hasExact()); } private List buildFieldSorts(List orders) { List sorts = new ArrayList<>(orders.size()); for (Order o : orders) { - sorts.add(new EsQueryExec.FieldSort(((FieldAttribute) o.child()).exactAttribute(), o.direction(), o.nullsPosition())); + sorts.add(new EsQueryExec.FieldSort(((FieldAttribute) o.child()), o.direction(), o.nullsPosition())); } return sorts; } @@ -308,10 +299,7 @@ private static final class EsqlTranslatorHandler extends QlTranslatorHandler { @Override public Query wrapFunctionQuery(ScalarFunction sf, Expression field, Supplier querySupplier) { if (field instanceof FieldAttribute fa) { - return ExpressionTranslator.wrapIfNested( - new SingleValueQuery(querySupplier.get(), fa.exactAttribute().name()), - ((FieldAttribute) field).exactAttribute() - ); + return ExpressionTranslator.wrapIfNested(new SingleValueQuery(querySupplier.get(), fa.name()), field); } if (field instanceof MetadataAttribute) { return querySupplier.get(); // MetadataAttributes are always single valued diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 768f51dab173a..9bce5fb25d739 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -12,7 +12,9 @@ import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.expression.SurrogateExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; @@ -28,6 +30,7 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.ExpressionSet; import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.Order; @@ -35,10 +38,7 @@ import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.ql.expression.predicate.Predicates; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; -import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; -import org.elasticsearch.xpack.ql.expression.predicate.regex.StringPattern; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BinaryComparisonSimplification; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanFunctionEqualsElimination; @@ -55,16 +55,12 @@ import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.ql.rule.RuleExecutor; -import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.CollectionUtils; import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -73,8 +69,6 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.ql.expression.Expressions.asAttributes; -import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.combineOr; -import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.splitOr; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.FoldNull; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateNullable; @@ -96,6 +90,7 @@ protected static List> rules() { var operators = new Batch<>( "Operator Optimization", + new ReplaceAttributesWithExact(), new CombineProjections(), new PruneEmptyPlans(), new PropagateEmptyRelation(), @@ -687,76 +682,14 @@ private static Project pushDownPastProject(UnaryPlan parent) { * This rule does NOT check for type compatibility as that phase has been * already be verified in the analyzer. */ - public static class CombineDisjunctionsToIn extends OptimizerRules.OptimizerExpressionRule { - public CombineDisjunctionsToIn() { - super(TransformDirection.UP); - } - - @Override - protected Expression rule(Or or) { - Expression e = or; - // look only at equals and In - List exps = splitOr(e); - - Map> found = new LinkedHashMap<>(); - ZoneId zoneId = null; - List ors = new LinkedList<>(); - - for (Expression exp : exps) { - if (exp instanceof Equals eq) { - // consider only equals against foldables - if (eq.right().foldable()) { - found.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right()); - } else { - ors.add(exp); - } - if (zoneId == null) { - zoneId = eq.zoneId(); - } - } else if (exp instanceof org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In in) { - found.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(in.list()); - if (zoneId == null) { - zoneId = in.zoneId(); - } - } else { - ors.add(exp); - } - } + public static class CombineDisjunctionsToIn extends OptimizerRules.CombineDisjunctionsToIn { - if (found.isEmpty() == false) { - // combine equals alongside the existing ors - final ZoneId finalZoneId = zoneId; - found.forEach((k, v) -> { - ors.add( - v.size() == 1 - ? new org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals( - k.source(), - k, - v.iterator().next(), - finalZoneId - ) - : createIn(k, new ArrayList<>(v), finalZoneId) - ); - }); - - Expression combineOr = combineOr(ors); - // check the result semantically since the result might different in order - // but be actually the same which can trigger a loop - // e.g. a == 1 OR a == 2 OR null --> null OR a in (1,2) --> literalsOnTheRight --> cycle - if (e.semanticEquals(combineOr) == false) { - e = combineOr; - } - } - - return e; + protected In createIn(Expression key, List values, ZoneId zoneId) { + return new In(key.source(), key, values); } - protected org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In createIn( - Expression key, - List values, - ZoneId zoneId - ) { - return new In(key.source(), key, values); + protected Equals createEquals(Expression k, Set v, ZoneId finalZoneId) { + return new Equals(k.source(), k, v.iterator().next(), finalZoneId); } } @@ -772,34 +705,52 @@ protected LogicalPlan rule(Limit plan) { } } - public static class ReplaceRegexMatch extends OptimizerRules.OptimizerExpressionRule> { + public static class ReplaceRegexMatch extends OptimizerRules.ReplaceRegexMatch { - public ReplaceRegexMatch() { - super(TransformDirection.DOWN); + protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { + return new org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals( + regexMatch.source(), + regexMatch.field(), + literal + ); } + } + + private static class ReplaceAttributesWithExact extends OptimizerRules.OptimizerRule { @Override - protected Expression rule(RegexMatch regexMatch) { - Expression e = regexMatch; - StringPattern pattern = regexMatch.pattern(); - if (pattern.matchesAll()) { - e = new IsNotNull(e.source(), regexMatch.field()); + protected LogicalPlan rule(LogicalPlan plan) { + if (plan instanceof Drop) { + return plan; + } else if (plan instanceof Project proj) { + return plan.transformExpressionsOnlyUp(FieldAttribute.class, x -> toExactAlias(x, proj.projections())); } else { - String match = pattern.exactMatch(); - if (match != null) { - Literal literal = new Literal(regexMatch.source(), match, DataTypes.KEYWORD); - e = regexToEquals(regexMatch, literal); - } + return plan.transformExpressionsOnly(FieldAttribute.class, ReplaceAttributesWithExact::toExact); + } + } + + private NamedExpression toExactAlias(FieldAttribute e, List projections) { + + if (e.getExactInfo().hasExact() && e.exactAttribute() != e) { + FieldAttribute calculatedExact = toExact(e); + + // avoid using multiple IDs just to load the same field twice + NamedExpression exact = projections.stream() + .filter(x -> x.name().equals(calculatedExact.name())) + .map(NamedExpression.class::cast) + .findFirst() + .orElse(calculatedExact); + + return new Alias(e.source(), e.name(), exact); } return e; } - protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { - return new org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals( - regexMatch.source(), - regexMatch.field(), - literal - ); + private static FieldAttribute toExact(FieldAttribute fa) { + if (fa.getExactInfo().hasExact() && fa.exactAttribute() != fa) { + return fa.exactAttribute(); + } + return fa; } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 5074b45f02428..8cda7a12fd0af 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -212,13 +212,13 @@ public void testProjectStar() { assertProjection(""" from test | keep * - """, "_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "salary"); + """, "_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "salary"); } public void testNoProjection() { assertProjection(""" from test - """, "_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "salary"); + """, "_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "salary"); assertProjectionTypes( """ from test @@ -227,6 +227,8 @@ public void testNoProjection() { DataTypes.INTEGER, DataTypes.KEYWORD, DataTypes.TEXT, + DataTypes.TEXT, + DataTypes.KEYWORD, DataTypes.INTEGER, DataTypes.KEYWORD, DataTypes.INTEGER @@ -237,7 +239,7 @@ public void testProjectOrder() { assertProjection(""" from test | keep first_name, *, last_name - """, "first_name", "_meta_field", "emp_no", "gender", "languages", "salary", "last_name"); + """, "first_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "salary", "last_name"); } public void testProjectThenDropName() { @@ -269,21 +271,21 @@ public void testProjectDropPattern() { from test | keep * | drop *_name - """, "_meta_field", "emp_no", "gender", "languages", "salary"); + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "salary"); } public void testProjectDropNoStarPattern() { assertProjection(""" from test | drop *_name - """, "_meta_field", "emp_no", "gender", "languages", "salary"); + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "salary"); } public void testProjectOrderPatternWithRest() { assertProjection(""" from test | keep *name, *, emp_no - """, "first_name", "last_name", "_meta_field", "gender", "languages", "salary", "emp_no"); + """, "first_name", "last_name", "_meta_field", "gender", "job", "job.raw", "languages", "salary", "emp_no"); } public void testProjectDropPatternAndKeepOthers() { @@ -420,7 +422,7 @@ public void testDropPatternUnsupportedFields() { assertProjection(""" from test | drop *ala* - """, "_meta_field", "emp_no", "first_name", "gender", "languages", "last_name"); + """, "_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name"); } public void testDropUnsupportedPattern() { @@ -488,7 +490,7 @@ public void testRenameReuseAlias() { assertProjection(""" from test | rename emp_no as e, first_name as e - """, "_meta_field", "e", "gender", "languages", "last_name", "salary"); + """, "_meta_field", "e", "gender", "job", "job.raw", "languages", "last_name", "salary"); } public void testRenameUnsupportedField() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 4810dcdedec00..f3c61d002a69e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -1278,6 +1278,43 @@ public void testSplittingInWithFoldableValue() { assertThat(new LogicalPlanOptimizer.SplitInWithFoldableValue().rule(in), equalTo(expected)); } + public void testReplaceFilterWithExact() { + var plan = plan(""" + from test + | where job == "foo" + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + Equals equals = as(filter.condition(), Equals.class); + FieldAttribute left = as(equals.left(), FieldAttribute.class); + assertThat(left.name(), equalTo("job.raw")); + } + + public void testReplaceExpressionWithExact() { + var plan = plan(""" + from test + | eval x = job + """); + + var eval = as(plan, Eval.class); + var alias = as(eval.fields().get(0), Alias.class); + var field = as(alias.child(), FieldAttribute.class); + assertThat(field.name(), equalTo("job.raw")); + } + + public void testReplaceSortWithExact() { + var plan = plan(""" + from test + | sort job + """); + + var topN = as(plan, TopN.class); + assertThat(topN.order().size(), equalTo(1)); + var sortField = as(topN.order().get(0).child(), FieldAttribute.class); + assertThat(sortField.name(), equalTo("job.raw")); + } + private LogicalPlan optimizedPlan(String query) { return plan(query); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 49e1fa1f15c16..5753c87e0f89b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -78,6 +78,7 @@ import org.elasticsearch.xpack.ql.type.EsField; import org.junit.Before; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -204,7 +205,10 @@ public void testSingleFieldExtractor() { var filter = as(limit.child(), FilterExec.class); var extract = as(filter.child(), FieldExtractExec.class); - assertEquals(Sets.difference(mapping.keySet(), Set.of("emp_no")), Sets.newHashSet(names(restExtract.attributesToExtract()))); + assertEquals( + Sets.difference(allFieldsExcludingTextWithExact(mapping), Set.of("emp_no")), + Sets.newHashSet(names(restExtract.attributesToExtract())) + ); assertEquals(Set.of("emp_no"), Sets.newHashSet(names(extract.attributesToExtract()))); var query = as(extract.child(), EsQueryExec.class); @@ -228,7 +232,10 @@ public void testExactlyOneExtractorPerFieldWithPruning() { var filter = as(limit.child(), FilterExec.class); var extract = as(filter.child(), FieldExtractExec.class); - assertEquals(Sets.difference(mapping.keySet(), Set.of("emp_no")), Sets.newHashSet(names(restExtract.attributesToExtract()))); + assertEquals( + Sets.difference(allFieldsExcludingTextWithExact(mapping), Set.of("emp_no")), + Sets.newHashSet(names(restExtract.attributesToExtract())) + ); assertThat(names(extract.attributesToExtract()), contains("emp_no")); var query = source(extract.child()); @@ -236,6 +243,22 @@ public void testExactlyOneExtractorPerFieldWithPruning() { assertThat(query.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); } + private Set allFieldsExcludingTextWithExact(Map mapping) { + Set result = new HashSet<>(); + for (Map.Entry entry : mapping.entrySet()) { + String name = entry.getKey(); + EsField field = entry.getValue(); + if (field.getDataType() != DataTypes.TEXT || field.getProperties().isEmpty()) { + result.add(name); + } else { + for (String s : entry.getValue().getProperties().keySet()) { + result.add(entry.getKey() + "." + s); + } + } + } + return result; + } + public void testDoubleExtractorPerFieldEvenWithAliasNoPruningDueToImplicitProjection() { var plan = physicalPlan(""" from test @@ -376,7 +399,7 @@ public void testExtractorMultiEvalWithDifferentNames() { var extract = as(project.child(), FieldExtractExec.class); assertThat( names(extract.attributesToExtract()), - contains("_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "salary") + contains("_meta_field", "emp_no", "first_name", "gender", "job.raw", "languages", "last_name", "salary") ); } @@ -406,7 +429,7 @@ public void testExtractorMultiEvalWithSameName() { var extract = as(project.child(), FieldExtractExec.class); assertThat( names(extract.attributesToExtract()), - contains("_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "salary") + contains("_meta_field", "emp_no", "first_name", "gender", "job.raw", "languages", "last_name", "salary") ); } @@ -865,7 +888,7 @@ public void testPushLimitAndFilterToSource() { assertThat( names(extract.attributesToExtract()), - contains("_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "salary") + contains("_meta_field", "emp_no", "first_name", "gender", "job.raw", "languages", "last_name", "salary") ); var source = source(extract.child()); @@ -1671,6 +1694,22 @@ public void testNoTextFilterPushDown() { assertNull(source.query()); } + public void testTextWithRawFilterPushDown() { + var plan = physicalPlan(""" + from test + | where job == "foo" + """); + + var optimized = optimizedPlan(plan); + var limit = as(optimized, LimitExec.class); + var exchange = asRemoteExchange(limit.child()); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + var source = as(extract.child(), EsQueryExec.class); + var qb = as(source.query(), SingleValueQuery.Builder.class); + assertThat(qb.field(), equalTo("job.raw")); + } + public void testNoTextSortPushDown() { var plan = physicalPlan(""" from test @@ -1688,6 +1727,22 @@ public void testNoTextSortPushDown() { assertNull(source.sorts()); } + public void testTextWithRawSortPushDown() { + var plan = physicalPlan(""" + from test + | sort job + """); + + var optimized = optimizedPlan(plan); + var topN = as(optimized, TopNExec.class); + var exchange = as(topN.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + var source = as(extract.child(), EsQueryExec.class); + assertThat(source.sorts().size(), equalTo(1)); + assertThat(source.sorts().get(0).field().name(), equalTo("job.raw")); + } + private static EsQueryExec source(PhysicalPlan plan) { if (plan instanceof ExchangeExec exchange) { plan = exchange.child(); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java index 74a16259635d4..58dd26b2331ba 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java @@ -1245,7 +1245,7 @@ protected Expression rule(Or or) { found.forEach((k, v) -> { ors.add( v.size() == 1 - ? new Equals(k.source(), k, v.iterator().next(), finalZoneId) + ? createEquals(k, v, finalZoneId) : createIn(k, new ArrayList<>(v), finalZoneId) ); }); @@ -1262,6 +1262,10 @@ protected Expression rule(Or or) { return e; } + protected Equals createEquals(Expression k, Set v, ZoneId finalZoneId) { + return new Equals(k.source(), k, v.iterator().next(), finalZoneId); + } + protected In createIn(Expression key, List values, ZoneId zoneId) { return new In(key.source(), key, values, zoneId); } From ffd27c37690260f259be92dbd8f378f7bb49d3aa Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 24 Aug 2023 09:40:36 +0200 Subject: [PATCH 06/10] Format QL code --- .../xpack/ql/optimizer/OptimizerRules.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java index c14b5d43245e6..76cf1b6690335 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java @@ -1242,13 +1242,9 @@ protected Expression rule(Or or) { if (found.isEmpty() == false) { // combine equals alongside the existing ors final ZoneId finalZoneId = zoneId; - found.forEach((k, v) -> { - ors.add( - v.size() == 1 - ? createEquals(k, v, finalZoneId) - : createIn(k, new ArrayList<>(v), finalZoneId) - ); - }); + found.forEach( + (k, v) -> { ors.add(v.size() == 1 ? createEquals(k, v, finalZoneId) : createIn(k, new ArrayList<>(v), finalZoneId)); } + ); Expression combineOr = combineOr(ors); // check the result semantically since the result might different in order From 27814b8df467d2b7532620866894a7dfee205115 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 24 Aug 2023 09:56:43 +0200 Subject: [PATCH 07/10] Update docs/changelog/98528.yaml --- docs/changelog/98528.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/98528.yaml diff --git a/docs/changelog/98528.yaml b/docs/changelog/98528.yaml new file mode 100644 index 0000000000000..d392942acd8bc --- /dev/null +++ b/docs/changelog/98528.yaml @@ -0,0 +1,5 @@ +pr: 98528 +summary: "ESQL: Add support for TEXT fields in comparison operators and SORT" +area: ES|QL +type: enhancement +issues: [] From 1496dbe574518434b560836140c64af67630d40a Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 24 Aug 2023 09:58:37 +0200 Subject: [PATCH 08/10] Update docs/changelog/98528.yaml --- docs/changelog/98528.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog/98528.yaml b/docs/changelog/98528.yaml index d392942acd8bc..0004499e58f83 100644 --- a/docs/changelog/98528.yaml +++ b/docs/changelog/98528.yaml @@ -2,4 +2,5 @@ pr: 98528 summary: "ESQL: Add support for TEXT fields in comparison operators and SORT" area: ES|QL type: enhancement -issues: [] +issues: + - 98642 From 8462246513e0c17ff49625d3ed2d3900eca0f58f Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 25 Aug 2023 12:43:03 +0200 Subject: [PATCH 09/10] Implement review suggestions --- .../resources/rest-api-spec/test/80_text.yml | 44 ++++++++++++---- .../predicate/operator/comparison/Equals.java | 10 ++-- .../operator/comparison/GreaterThan.java | 10 ++-- .../comparison/GreaterThanOrEqual.java | 10 ++-- .../operator/comparison/LessThan.java | 10 ++-- .../operator/comparison/LessThanOrEqual.java | 10 ++-- .../operator/comparison/NotEquals.java | 10 ++-- .../esql/expression/EsqlTypeResolutions.java | 45 ++++++++++++++++ .../expression/function/aggregate/Count.java | 5 +- .../function/aggregate/CountDistinct.java | 4 +- .../operator/comparison/NullEquals.java | 10 ++-- .../esql/optimizer/LogicalPlanOptimizer.java | 38 ++++---------- .../optimizer/LogicalPlanOptimizerTests.java | 2 +- .../optimizer/PhysicalPlanOptimizerTests.java | 52 +++++++++---------- 14 files changed, 150 insertions(+), 110 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml index 2e197cc364da7..7f8b5d5495195 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml @@ -28,6 +28,30 @@ setup: - { "emp_no": 10, "name": "Jenny", "job": "IT Director", "tag": "foo bar" } - { "index": { } } - { "emp_no": 20, "name": "John", "job": "Payroll Specialist", "tag": "baz" } + +--- +"all": + - do: + esql.query: + body: + query: 'from test | sort emp_no' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "job" } + - match: { columns.1.type: "text" } + - match: { columns.2.name: "job.raw" } + - match: { columns.2.type: "keyword" } + - match: { columns.3.name: "name" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "tag" } + - match: { columns.4.type: "text" } + + - length: { values: 2 } + - match: { values.0: [ 10, "IT Director", "IT Director", "Jenny", "foo bar"] } + - match: { values.1: [ 20, "Payroll Specialist", "Payroll Specialist", "John", "baz"] } + + --- "filter by text": - do: @@ -40,7 +64,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -59,7 +83,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -78,7 +102,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -97,7 +121,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "x" } - match: { columns.3.type: "text" } @@ -116,7 +140,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -135,7 +159,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -154,7 +178,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -174,7 +198,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -195,7 +219,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } @@ -215,7 +239,7 @@ setup: - match: { columns.1.name: "name" } - match: { columns.1.type: "keyword" } - match: { columns.2.name: "job" } - - match: { columns.2.type: "keyword" } + - match: { columns.2.type: "text" } - match: { columns.3.name: "tag" } - match: { columns.3.type: "text" } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java index c9efed3069c4b..db3822f047573 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java @@ -8,16 +8,17 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class Equals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals { public Equals(Source source, Expression left, Expression right) { super(source, left, right); @@ -29,10 +30,7 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { - return TypeResolution.TYPE_RESOLVED; - } - return super.resolveInputType(e, paramOrdinal); + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java index 4d51e3db945c4..5683a9d0d7e85 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java @@ -8,16 +8,17 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class GreaterThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan { public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { super(source, left, right, zoneId); @@ -25,10 +26,7 @@ public GreaterThan(Source source, Expression left, Expression right, ZoneId zone @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { - return TypeResolution.TYPE_RESOLVED; - } - return super.resolveInputType(e, paramOrdinal); + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java index 59456860e74b0..ebb29998fb995 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java @@ -8,16 +8,17 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class GreaterThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual { public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { @@ -26,10 +27,7 @@ public GreaterThanOrEqual(Source source, Expression left, Expression right, Zone @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { - return TypeResolution.TYPE_RESOLVED; - } - return super.resolveInputType(e, paramOrdinal); + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java index 3580bf50918e8..12f54270b65dc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java @@ -8,16 +8,17 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class LessThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan { public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) { @@ -26,10 +27,7 @@ public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { - return TypeResolution.TYPE_RESOLVED; - } - return super.resolveInputType(e, paramOrdinal); + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java index 7dabd4aa77b05..e75733a9e2340 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java @@ -8,16 +8,17 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class LessThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual { public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { super(source, left, right, zoneId); @@ -25,10 +26,7 @@ public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { - return TypeResolution.TYPE_RESOLVED; - } - return super.resolveInputType(e, paramOrdinal); + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java index e8e81d3957e9c..67319bab11b19 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java @@ -8,16 +8,17 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class NotEquals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals { public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { super(source, left, right, zoneId); @@ -25,10 +26,7 @@ public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { - return TypeResolution.TYPE_RESOLVED; - } - return super.resolveInputType(e, paramOrdinal); + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java new file mode 100644 index 0000000000000..d47ccf11c9985 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.type.EsField; + +import java.util.Locale; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + +public class EsqlTypeResolutions { + + public static Expression.TypeResolution isExact(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { + if (e instanceof FieldAttribute fa) { + if (DataTypes.isString(fa.dataType())) { + // ESQL can extract exact values for TEXT fields + return Expression.TypeResolution.TYPE_RESOLVED; + } + EsField.Exact exact = fa.getExactInfo(); + if (exact.hasExact() == false) { + return new Expression.TypeResolution( + format( + null, + "[{}] cannot operate on {}field of data type [{}]: {}", + operationName, + paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " argument ", + e.dataType().typeName(), + exact.errorMsg() + ) + ); + } + } + return Expression.TypeResolution.TYPE_RESOLVED; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 26adbb255b0c7..959907ef93257 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.CountAggregatorFunction; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.esql.planner.ToAggregator; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Nullability; @@ -22,6 +23,8 @@ import java.util.List; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class Count extends AggregateFunction implements EnclosedAgg, ToAggregator { public Count(Source source, Expression field) { @@ -60,6 +63,6 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return field().dataType() == DataTypes.TEXT ? TypeResolution.TYPE_RESOLVED : super.resolveType(); + return EsqlTypeResolutions.isExact(field(), sourceText(), DEFAULT); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index b25db3c8628ee..bbe00ffafc524 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -15,6 +15,7 @@ import org.elasticsearch.compute.aggregation.CountDistinctIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.CountDistinctLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlUnsupportedOperationException; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.esql.planner.ToAggregator; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.function.OptionalArgument; @@ -26,6 +27,7 @@ import java.util.List; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isInteger; @@ -59,7 +61,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = field().dataType() == DataTypes.TEXT ? TypeResolution.TYPE_RESOLVED : super.resolveType(); + TypeResolution resolution = EsqlTypeResolutions.isExact(field(), sourceText(), DEFAULT); if (resolution.unresolved() || precision == null) { return resolution; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java index 9e2b45967883c..4ab2d3fa8e7b9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NullEquals.java @@ -7,15 +7,16 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + public class NullEquals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals { public NullEquals(Source source, Expression left, Expression right, ZoneId zoneId) { super(source, left, right, zoneId); @@ -23,10 +24,7 @@ public NullEquals(Source source, Expression left, Expression right, ZoneId zoneI @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - if (e instanceof FieldAttribute fa && fa.dataType() == DataTypes.TEXT) { - return TypeResolution.TYPE_RESOLVED; - } - return super.resolveInputType(e, paramOrdinal); + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index fb36ca0c2d9d0..56ce90c3475b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -14,7 +14,6 @@ import org.elasticsearch.xpack.esql.expression.SurrogateExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; -import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; @@ -88,11 +87,16 @@ protected List> batches() { } protected static List> rules() { - var substitutions = new Batch<>("Substitutions", Limiter.ONCE, new SubstituteSurrogates(), new ReplaceRegexMatch()); + var substitutions = new Batch<>( + "Substitutions", + Limiter.ONCE, + new SubstituteSurrogates(), + new ReplaceRegexMatch(), + new ReplaceFieldAttributesWithExactSubfield() + ); var operators = new Batch<>( "Operator Optimization", - new ReplaceAttributesWithExact(), new CombineProjections(), new PruneEmptyPlans(), new PropagateEmptyRelation(), @@ -832,34 +836,14 @@ protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { } } - private static class ReplaceAttributesWithExact extends OptimizerRules.OptimizerRule { + private static class ReplaceFieldAttributesWithExactSubfield extends OptimizerRules.OptimizerRule { @Override protected LogicalPlan rule(LogicalPlan plan) { - if (plan instanceof LocalRelation || plan instanceof Drop) { - return plan; - } else if (plan instanceof Project proj) { - return plan.transformExpressionsOnlyUp(FieldAttribute.class, x -> toExactAlias(x, proj.projections())); - } else { - return plan.transformExpressionsOnly(FieldAttribute.class, ReplaceAttributesWithExact::toExact); - } - } - - private NamedExpression toExactAlias(FieldAttribute e, List projections) { - - if (e.getExactInfo().hasExact() && e.exactAttribute() != e) { - FieldAttribute calculatedExact = toExact(e); - - // avoid using multiple IDs just to load the same field twice - NamedExpression exact = projections.stream() - .filter(x -> x.name().equals(calculatedExact.name())) - .map(NamedExpression.class::cast) - .findFirst() - .orElse(calculatedExact); - - return new Alias(e.source(), e.name(), exact); + if (plan instanceof Filter || plan instanceof OrderBy || plan instanceof Aggregate) { + return plan.transformExpressionsOnly(FieldAttribute.class, ReplaceFieldAttributesWithExactSubfield::toExact); } - return e; + return plan; } private static FieldAttribute toExact(FieldAttribute fa) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 2ebee25fbba00..b6d818038a72c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -1449,7 +1449,7 @@ public void testReplaceExpressionWithExact() { var eval = as(plan, Eval.class); var alias = as(eval.fields().get(0), Alias.class); var field = as(alias.child(), FieldAttribute.class); - assertThat(field.name(), equalTo("job.raw")); + assertThat(field.name(), equalTo("job")); } public void testReplaceSortWithExact() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index ec62533f466d3..69c2ea900289e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -156,7 +156,13 @@ public void init() { mapping = loadMapping("mapping-basic.json"); allFieldRowSize = mapping.values() .stream() - .mapToInt(f -> EstimatesRowSize.estimateSize(EsqlDataTypes.widenSmallNumericTypes(f.getDataType()))) + .mapToInt( + f -> (EstimatesRowSize.estimateSize(EsqlDataTypes.widenSmallNumericTypes(f.getDataType())) + f.getProperties() + .values() + .stream() + .mapToInt(x -> EstimatesRowSize.estimateSize(EsqlDataTypes.widenSmallNumericTypes(x.getDataType()))) + .sum()) + ) .sum(); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); @@ -206,16 +212,25 @@ public void testSingleFieldExtractor() { var filter = as(limit.child(), FilterExec.class); var extract = as(filter.child(), FieldExtractExec.class); - assertEquals( - Sets.difference(allFieldsExcludingTextWithExact(mapping), Set.of("emp_no")), - Sets.newHashSet(names(restExtract.attributesToExtract())) - ); + assertEquals(Sets.difference(allFields(mapping), Set.of("emp_no")), Sets.newHashSet(names(restExtract.attributesToExtract()))); assertEquals(Set.of("emp_no"), Sets.newHashSet(names(extract.attributesToExtract()))); var query = as(extract.child(), EsQueryExec.class); assertThat(query.estimatedRowSize(), equalTo(Integer.BYTES + allFieldRowSize)); } + private Set allFields(Map mapping) { + Set result = new HashSet<>(); + for (Map.Entry entry : mapping.entrySet()) { + String key = entry.getKey(); + result.add(key); + for (Map.Entry sub : entry.getValue().getProperties().entrySet()) { + result.add(key + "." + sub.getKey()); + } + } + return result; + } + public void testExactlyOneExtractorPerFieldWithPruning() { var plan = physicalPlan(""" from test @@ -233,10 +248,7 @@ public void testExactlyOneExtractorPerFieldWithPruning() { var filter = as(limit.child(), FilterExec.class); var extract = as(filter.child(), FieldExtractExec.class); - assertEquals( - Sets.difference(allFieldsExcludingTextWithExact(mapping), Set.of("emp_no")), - Sets.newHashSet(names(restExtract.attributesToExtract())) - ); + assertEquals(Sets.difference(allFields(mapping), Set.of("emp_no")), Sets.newHashSet(names(restExtract.attributesToExtract()))); assertThat(names(extract.attributesToExtract()), contains("emp_no")); var query = source(extract.child()); @@ -244,22 +256,6 @@ public void testExactlyOneExtractorPerFieldWithPruning() { assertThat(query.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); } - private Set allFieldsExcludingTextWithExact(Map mapping) { - Set result = new HashSet<>(); - for (Map.Entry entry : mapping.entrySet()) { - String name = entry.getKey(); - EsField field = entry.getValue(); - if (field.getDataType() != DataTypes.TEXT || field.getProperties().isEmpty()) { - result.add(name); - } else { - for (String s : entry.getValue().getProperties().keySet()) { - result.add(entry.getKey() + "." + s); - } - } - } - return result; - } - public void testDoubleExtractorPerFieldEvenWithAliasNoPruningDueToImplicitProjection() { var plan = physicalPlan(""" from test @@ -393,7 +389,7 @@ public void testExtractorMultiEvalWithDifferentNames() { var extract = as(project.child(), FieldExtractExec.class); assertThat( names(extract.attributesToExtract()), - contains("_meta_field", "emp_no", "first_name", "gender", "job.raw", "languages", "last_name", "salary") + contains("_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "salary") ); } @@ -423,7 +419,7 @@ public void testExtractorMultiEvalWithSameName() { var extract = as(project.child(), FieldExtractExec.class); assertThat( names(extract.attributesToExtract()), - contains("_meta_field", "emp_no", "first_name", "gender", "job.raw", "languages", "last_name", "salary") + contains("_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "salary") ); } @@ -880,7 +876,7 @@ public void testPushLimitAndFilterToSource() { assertThat( names(extract.attributesToExtract()), - contains("_meta_field", "emp_no", "first_name", "gender", "job.raw", "languages", "last_name", "salary") + contains("_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "salary") ); var source = source(extract.child()); From 5bfd34164a420ea004b1357d2ed32e310b220f47 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Tue, 29 Aug 2023 12:15:45 +0200 Subject: [PATCH 10/10] Add comments --- .../xpack/esql/optimizer/PhysicalPlanOptimizerTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 69c2ea900289e..1a5aa81eea418 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -160,6 +160,7 @@ public void init() { f -> (EstimatesRowSize.estimateSize(EsqlDataTypes.widenSmallNumericTypes(f.getDataType())) + f.getProperties() .values() .stream() + // check one more level since the mapping contains TEXT fields with KEYWORD multi-fields .mapToInt(x -> EstimatesRowSize.estimateSize(EsqlDataTypes.widenSmallNumericTypes(x.getDataType()))) .sum()) )