From 159c576e136ff2c0ec5951ccd702555e75f1f4ae Mon Sep 17 00:00:00 2001 From: afoucret Date: Thu, 2 Oct 2025 12:58:34 +0200 Subject: [PATCH 01/15] Add capability and transport version for KQL function optional parameters --- .../main/java/org/elasticsearch/TransportVersions.java | 1 + .../xpack/esql/action/EsqlCapabilities.java | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index ca43951a91b39..dcd8d2241b582 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -260,6 +260,7 @@ static TransportVersion def(int id) { public static final TransportVersion STATE_PARAM_GET_SNAPSHOT = def(9_100_0_00); public static final TransportVersion PROJECT_ID_IN_SNAPSHOTS_DELETIONS_AND_REPO_CLEANUP = def(9_101_0_00); public static final TransportVersion CLUSTER_STATE_PROJECTS_SETTINGS = def(9_108_0_00); + public static final TransportVersion ESQL_KQL_FUNCTION_OPTIONS = def(9_109_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 81bc6a5d89a0f..b7de40eba3416 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -729,6 +729,11 @@ public enum Cap { */ KQL_FUNCTION, + /** + * Support for optional parameters in KQL function (case_insensitive, time_zone, default_field, boost). + */ + KQL_FUNCTION_OPTIONS, + /** * Hash function */ @@ -895,11 +900,6 @@ public enum Cap { /** * Full text functions can be used in disjunctions */ - FULL_TEXT_FUNCTIONS_DISJUNCTIONS, - - /** - * Change field caps response for semantic_text fields to be reported as text - */ SEMANTIC_TEXT_FIELD_CAPS, /** From e0d6d42688600196a39f53ba63f5b918aae760cf Mon Sep 17 00:00:00 2001 From: afoucret Date: Thu, 2 Oct 2025 15:47:45 +0200 Subject: [PATCH 02/15] Implementing optional named params for the KQL function. --- .../esql/_snippets/functions/examples/kql.md | 6 +- .../functions/functionNamedParams/kql.md | 16 ++ .../esql/_snippets/functions/layout/kql.md | 3 + .../_snippets/functions/parameters/kql.md | 3 + .../esql/_snippets/functions/types/kql.md | 10 +- .../esql/images/functions/kql.svg | 2 +- .../esql/kibana/definition/functions/kql.json | 41 ++++- .../org/elasticsearch/TransportVersions.java | 1 - .../function/EsqlFunctionRegistry.java | 2 +- .../expression/function/fulltext/Kql.java | 145 ++++++++++++++++-- .../xpack/esql/querydsl/query/KqlQuery.java | 54 +++---- .../function/fulltext/KqlErrorTests.java | 40 ++++- .../function/fulltext/KqlTests.java | 63 +++++++- .../esql/querydsl/query/KqlQueryTests.java | 73 +++++---- 14 files changed, 369 insertions(+), 90 deletions(-) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md index 862d8f050e87e..5679b9bca07c0 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md @@ -1,6 +1,6 @@ % This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. -**Example** +**Examples** ```esql FROM books @@ -15,4 +15,8 @@ FROM books | 2883 | William Faulkner | | 3293 | Danny Faulkner | +```esql +null +``` + diff --git a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md new file mode 100644 index 0000000000000..bfa78e253feba --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md @@ -0,0 +1,16 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported function named parameters** + +`boost` +: (float) Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0. + +`time_zone` +: (keyword) Time zone to use when parsing date values. Defaults to UTC. + +`case_insensitive` +: (boolean) If true, the query is case insensitive. Defaults to false. + +`default_field` +: (keyword) Default field to search when no field is specified in the query. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md b/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md index 94a156bdc8a8a..20515c96854ee 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md @@ -22,5 +22,8 @@ stack: preview 9.0.0, ga 9.1.0 :::{include} ../types/kql.md ::: +:::{include} ../functionNamedParams/kql.md +::: + :::{include} ../examples/kql.md ::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md index 7532069aa5ed2..7e3c5fe2bec20 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md @@ -5,3 +5,6 @@ `query` : Query string in KQL query string format. +`options` +: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/kql.md b/docs/reference/query-languages/esql/_snippets/functions/types/kql.md index 0af3d49fd7399..fca2723a34442 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/kql.md @@ -2,8 +2,10 @@ **Supported types** -| query | result | -| --- | --- | -| keyword | boolean | -| text | boolean | +| query | options | result | +| --- | --- | --- | +| keyword | named parameters | boolean | +| keyword | | boolean | +| text | named parameters | boolean | +| text | | boolean | diff --git a/docs/reference/query-languages/esql/images/functions/kql.svg b/docs/reference/query-languages/esql/images/functions/kql.svg index 0700b1bf2ce1c..3b860bd88786b 100644 --- a/docs/reference/query-languages/esql/images/functions/kql.svg +++ b/docs/reference/query-languages/esql/images/functions/kql.svg @@ -1 +1 @@ -KQL(query) \ No newline at end of file +KQL(query,options) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json index f3fe7c2df4c11..7493f5dc35499 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json @@ -16,6 +16,25 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in KQL query string format." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, America/New_York], description='Time zone to use when parsing date values. Defaults to UTC.'}, {name='case_insensitive', values=[true, false], description='If true, the query is case insensitive. Defaults to false.'}, {name='default_field', values=[_all, title], description='Default field to search when no field is specified in the query.'}", + "optional" : true, + "description" : "(Optional) KQL additional options as <>." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -27,10 +46,30 @@ ], "variadic" : false, "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in KQL query string format." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, America/New_York], description='Time zone to use when parsing date values. Defaults to UTC.'}, {name='case_insensitive', values=[true, false], description='If true, the query is case insensitive. Defaults to false.'}, {name='default_field', values=[_all, title], description='Default field to search when no field is specified in the query.'}", + "optional" : true, + "description" : "(Optional) KQL additional options as <>." + } + ], + "variadic" : false, + "returnType" : "boolean" } ], "examples" : [ - "FROM books\n| WHERE KQL(\"author: Faulkner\")" + "FROM books\n| WHERE KQL(\"author: Faulkner\")", + null ], "preview" : false, "snapshot_only" : false diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index dcd8d2241b582..ca43951a91b39 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -260,7 +260,6 @@ static TransportVersion def(int id) { public static final TransportVersion STATE_PARAM_GET_SNAPSHOT = def(9_100_0_00); public static final TransportVersion PROJECT_ID_IN_SNAPSHOTS_DELETIONS_AND_REPO_CLEANUP = def(9_101_0_00); public static final TransportVersion CLUSTER_STATE_PROJECTS_SETTINGS = def(9_108_0_00); - public static final TransportVersion ESQL_KQL_FUNCTION_OPTIONS = def(9_109_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 077c9aaa2b7b6..8553b3f587372 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -504,7 +504,7 @@ private static FunctionDefinition[][] functions() { // fulltext functions new FunctionDefinition[] { def(Decay.class, quad(Decay::new), "decay"), - def(Kql.class, uni(Kql::new), "kql"), + def(Kql.class, bi(Kql::new), "kql"), def(Knn.class, tri(Knn::new), "knn"), def(Match.class, tri(Match::new), "match"), def(MultiMatch.class, MultiMatch::new, "multi_match"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index a9eb30ab3e52e..b7a685dd175ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -12,37 +12,65 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.Foldables; -import org.elasticsearch.xpack.esql.expression.function.Example; -import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; -import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; -import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; -import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.*; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Map.entry; +import static org.elasticsearch.index.query.AbstractQueryBuilder.BOOST_FIELD; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.expression.Foldables.TypeResolutionValidator.forPreOptimizationValidation; +import static org.elasticsearch.xpack.esql.expression.Foldables.resolveTypeQuery; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.CASE_INSENSITIVE_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.DEFAULT_FIELD_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.TIME_ZONE_FIELD; /** * Full text function that performs a {@link KqlQuery} . */ -public class Kql extends FullTextFunction { +public class Kql extends FullTextFunction implements OptionalArgument { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::readFrom); + // Options for KQL function. They don't need to be serialized as the data nodes will retrieve them from the query builder + private final transient Expression options; + + public static final Map ALLOWED_OPTIONS = Map.ofEntries( + entry(BOOST_FIELD.getPreferredName(), FLOAT), + entry(CASE_INSENSITIVE_FIELD.getPreferredName(), BOOLEAN), + entry(TIME_ZONE_FIELD.getPreferredName(), KEYWORD), + entry(DEFAULT_FIELD_FIELD.getPreferredName(), KEYWORD) + ); + @FunctionInfo( returnType = "boolean", appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.0.0"), @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.GA, version = "9.1.0") }, description = "Performs a KQL query. Returns true if the provided KQL query string matches the row.", - examples = { @Example(file = "kql-function", tag = "kql-with-field") } + examples = { @Example(file = "kql-function", tag = "kql-with-field"), @Example(file = "kql-function", tag = "kql-with-options") } ) public Kql( Source source, @@ -50,13 +78,44 @@ public Kql( name = "query", type = { "keyword", "text" }, description = "Query string in KQL query string format." - ) Expression queryString + ) Expression queryString, + @MapParam( + name = "options", + description = "(Optional) KQL additional options as <>.", + params = { + @MapParam.MapParamEntry( + name = "case_insensitive", + type = "boolean", + valueHint = { "true", "false" }, + description = "If true, the query is case insensitive. Defaults to false." + ), + @MapParam.MapParamEntry( + name = "time_zone", + type = "keyword", + valueHint = { "UTC", "America/New_York" }, + description = "Time zone to use when parsing date values. Defaults to UTC." + ), + @MapParam.MapParamEntry( + name = "default_field", + type = "keyword", + valueHint = { "_all", "title" }, + description = "Default field to search when no field is specified in the query." + ), + @MapParam.MapParamEntry( + name = "boost", + type = "float", + valueHint = { "2.5" }, + description = "Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0." + ) }, + optional = true + ) Expression options ) { - super(source, queryString, List.of(queryString), null); + this(source, queryString, options, null); } - public Kql(Source source, Expression queryString, QueryBuilder queryBuilder) { - super(source, queryString, List.of(queryString), queryBuilder); + public Kql(Source source, Expression queryString, Expression options, QueryBuilder queryBuilder) { + super(source, queryString, options == null ? List.of(queryString) : List.of(queryString, options), queryBuilder); + this.options = options; } private static Kql readFrom(StreamInput in) throws IOException { @@ -66,7 +125,8 @@ private static Kql readFrom(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); } - return new Kql(source, query, queryBuilder); + // Options are not serialized - they're embedded in the QueryBuilder + return new Kql(source, query, null, queryBuilder); } @Override @@ -76,6 +136,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { out.writeOptionalNamedWriteable(queryBuilder()); } + // Options are not serialized - they're embedded in the QueryBuilder } @Override @@ -83,23 +144,75 @@ public String getWriteableName() { return ENTRY.name; } + public Expression options() { + return options; + } + + private TypeResolution resolveQuery() { + TypeResolution result = isType(query(), t -> t == KEYWORD || t == TEXT, sourceText(), FIRST, "keyword, text").and( + isNotNull(query(), sourceText(), FIRST) + ); + if (result.unresolved()) { + return result; + } + result = resolveTypeQuery(query(), sourceText(), forPreOptimizationValidation(query())); + if (result.equals(TypeResolution.TYPE_RESOLVED) == false) { + return result; + } + return TypeResolution.TYPE_RESOLVED; + } + + @Override + protected TypeResolution resolveParams() { + return resolveQuery().and(Options.resolve(options(), source(), SECOND, ALLOWED_OPTIONS)); + } + + private Map kqlQueryOptions() throws InvalidArgumentException { + if (options() == null) { + return null; + } + + Map kqlOptions = new HashMap<>(); + Options.populateMap((MapExpression) options(), kqlOptions, source(), SECOND, ALLOWED_OPTIONS); + return kqlOptions; + } + @Override public Expression replaceChildren(List newChildren) { - return new Kql(source(), newChildren.get(0), queryBuilder()); + return new Kql(source(), newChildren.get(0), newChildren.size() > 1 ? newChildren.get(1) : null, queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Kql::new, query(), queryBuilder()); + return NodeInfo.create(this, Kql::new, query(), options(), queryBuilder()); } @Override protected Query translate(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { - return new KqlQuery(source(), Foldables.queryAsString(query(), sourceText())); + return new KqlQuery(source(), Foldables.queryAsString(query(), sourceText()), kqlQueryOptions()); } @Override public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { - return new Kql(source(), query(), queryBuilder); + return new Kql(source(), query(), options(), queryBuilder); + } + + @Override + public boolean equals(Object o) { + // KQL does not serialize options, as they get included in the query builder. We need to override equals and hashcode to + // ignore options when comparing. + if (o == null || getClass() != o.getClass()) return false; + var kql = (Kql) o; + return Objects.equals(query(), kql.query()) && Objects.equals(queryBuilder(), kql.queryBuilder()); + } + + @Override + public int hashCode() { + return Objects.hash(query(), queryBuilder()); + } + + @Override + public String toString() { + return "Kql{" + "query=" + query() + (options == null ? "" : ", options=" + options) + '}'; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java index 56229ec325d73..30908fe3d8cbe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.esql.querydsl.query; -import org.elasticsearch.core.Booleans; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -18,28 +17,27 @@ import java.util.function.BiConsumer; import static java.util.Map.entry; +import static org.elasticsearch.index.query.AbstractQueryBuilder.BOOST_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.CASE_INSENSITIVE_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.DEFAULT_FIELD_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.TIME_ZONE_FIELD; public class KqlQuery extends Query { - private static final Map> BUILDER_APPLIERS = Map.ofEntries( - entry(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), KqlQueryBuilder::timeZone), - entry(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), KqlQueryBuilder::defaultField), - entry(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), (qb, s) -> qb.caseInsensitive(Booleans.parseBoolean(s))) + private static final Map> BUILDER_APPLIERS = Map.ofEntries( + entry(TIME_ZONE_FIELD.getPreferredName(), (qb, v) -> qb.timeZone((String) v)), + entry(DEFAULT_FIELD_FIELD.getPreferredName(), (qb, v) -> qb.defaultField((String) v)), + entry(CASE_INSENSITIVE_FIELD.getPreferredName(), (qb, v) -> qb.caseInsensitive((Boolean) v)), + entry(BOOST_FIELD.getPreferredName(), (qb, v) -> qb.boost(((Number) v).floatValue())) ); private final String query; + private final Map options; - private final Map options; - - // dedicated constructor for QueryTranslator - public KqlQuery(Source source, String query) { - this(source, query, null); - } - - public KqlQuery(Source source, String query, Map options) { + public KqlQuery(Source source, String query, Map options) { super(source); this.query = query; - this.options = options == null ? Collections.emptyMap() : options; + this.options = options == null ? Collections.emptyMap() : Map.copyOf(options); } @Override @@ -55,14 +53,24 @@ protected QueryBuilder asBuilder() { return queryBuilder; } + @Override + public boolean containsPlan() { + return false; + } + public String query() { return query; } - public Map options() { + public Map options() { return options; } + @Override + protected String innerToString() { + return query; + } + @Override public int hashCode() { return Objects.hash(query, options); @@ -73,23 +81,7 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - KqlQuery other = (KqlQuery) obj; return Objects.equals(query, other.query) && Objects.equals(options, other.options); } - - @Override - protected String innerToString() { - return query; - } - - @Override - public boolean scorable() { - return true; - } - - @Override - public boolean containsPlan() { - return false; - } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java index 891c419841e70..8377ce664a345 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java @@ -8,16 +8,21 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.hamcrest.Matcher; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.stream.Stream; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.hamcrest.Matchers.equalTo; public class KqlErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -34,11 +39,42 @@ protected Stream> testCandidates(List cases, Se @Override protected Expression build(Source source, List args) { - return new Kql(source, args.get(0)); + return new Kql(source, args.getFirst(), args.size() > 1 ? args.get(1) : null); } @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + return equalTo(errorMessageStringForKql(validPerPosition, signature, (l, p) -> "keyword, text")); + } + + private static String errorMessageStringForKql( + List> validPerPosition, + List signature, + AbstractFunctionTestCase.PositionalErrorMessageSupplier positionalErrorMessageSupplier + ) { + boolean invalid = false; + for (int i = 0; i < signature.size() && invalid == false; i++) { + // Need to check for nulls and bad parameters in order + if (signature.get(i) == DataType.NULL) { + return TypeResolutions.ParamOrdinal.fromIndex(i).name().toLowerCase(Locale.ROOT) + + " argument of [" + + sourceForSignature(signature) + + "] cannot be null, received []"; + } + if (validPerPosition.get(i).contains(signature.get(i)) == false) { + // Map expressions have different error messages + if (i == 1) { + return format(null, "second argument of [{}] must be a map expression, received []", sourceForSignature(signature)); + } + break; + } + } + + try { + return typeErrorMessage(true, validPerPosition, signature, positionalErrorMessageSupplier); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is for nulls or from the combination + return EsqlBinaryComparison.formatIncompatibleTypesMessage(signature.get(0), signature.get(1), sourceForSignature(signature)); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java index 118bf56c5d6b9..db2ada15ab49f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java @@ -10,13 +10,26 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; +import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize; +import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; +import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; +import static org.hamcrest.Matchers.equalTo; + public class KqlTests extends NoneFieldFullTextFunctionTestCase { public KqlTests(@Name("TestCase") Supplier testCaseSupplier) { super(testCaseSupplier); @@ -24,11 +37,57 @@ public KqlTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - return generateParameters(); + return parameterSuppliersFromTypedData(addFunctionNamedParams(getStringTestSupplier())); + } + + /** + * Adds function named parameters to all the test case suppliers provided + */ + private static List addFunctionNamedParams(List suppliers) { + List result = new ArrayList<>(suppliers); + for (TestCaseSupplier supplier : suppliers) { + List dataTypes = new ArrayList<>(supplier.types()); + dataTypes.add(UNSUPPORTED); + result.add(new TestCaseSupplier(supplier.name() + ", options", dataTypes, () -> { + List values = new ArrayList<>(supplier.get().getData()); + values.add( + new TestCaseSupplier.TypedData( + new MapExpression(Source.EMPTY, List.of(Literal.keyword(Source.EMPTY, "case_insensitive"), Literal.TRUE)), + UNSUPPORTED, + "options" + ).forceLiteral() + ); + + return new TestCaseSupplier.TestCase(values, equalTo("KqlEvaluator"), BOOLEAN, equalTo(true)); + })); + } + return result; } @Override protected Expression build(Source source, List args) { - return new Kql(source, args.get(0)); + Kql kql = new Kql(source, args.get(0), args.size() > 1 ? args.get(1) : null); + // We need to add the QueryBuilder to the kql expression, as it is used to implement equals() and hashCode() and + // thus test the serialization methods. But we can only do this if the parameters make sense. + if (args.get(0).foldable()) { + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, kql).toQueryBuilder(); + kql = (Kql) kql.replaceQueryBuilder(queryBuilder); + } + return kql; + } + + /** + * Copy of the overridden method that doesn't check for children size, as the {@code options} child isn't serialized in Kql. + */ + @Override + protected Expression serializeDeserializeExpression(Expression expression) { + Expression newExpression = serializeDeserialize( + expression, + PlanStreamOutput::writeNamedWriteable, + in -> in.readNamedWriteable(Expression.class), + testCase.getConfiguration() // The configuration query should be == to the source text of the function for this to work + ); + // Fields use synthetic sources, which can't be serialized. So we use the originals instead. + return newExpression.replaceChildren(expression.children()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java index 8dfb50f84ac1e..0ea107b1ef7e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.querydsl.query; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.SourceTests; @@ -14,32 +16,36 @@ import java.time.ZoneId; import java.time.zone.ZoneRulesException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.index.query.AbstractQueryBuilder.BOOST_FIELD; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.CASE_INSENSITIVE_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.DEFAULT_FIELD_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.TIME_ZONE_FIELD; import static org.hamcrest.Matchers.equalTo; public class KqlQueryTests extends ESTestCase { static KqlQuery randomKqkQueryQuery() { - Map options = new HashMap<>(); + Map options = new HashMap<>(); if (randomBoolean()) { - options.put(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), String.valueOf(randomBoolean())); + options.put(CASE_INSENSITIVE_FIELD.getPreferredName(), randomBoolean()); } if (randomBoolean()) { - options.put(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), randomIdentifier()); + options.put(DEFAULT_FIELD_FIELD.getPreferredName(), randomIdentifier()); } if (randomBoolean()) { - options.put(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), randomZone().getId()); + options.put(TIME_ZONE_FIELD.getPreferredName(), randomZone().getId()); + } + + if (randomBoolean()) { + options.put(BOOST_FIELD.getPreferredName(), randomFloat() * 5.0f + 0.1f); // Random float between 0.1 and 5.1 } return new KqlQuery(SourceTests.randomSource(), randomAlphaOfLength(5), Collections.unmodifiableMap(options)); @@ -65,8 +71,8 @@ private static KqlQuery mutate(KqlQuery query) { return randomFrom(options).apply(query); } - private static Map mutateOptions(Map options) { - Map mutatedOptions = new HashMap<>(options); + private static Map mutateOptions(Map options) { + Map mutatedOptions = new HashMap<>(options); if (options.isEmpty() == false && randomBoolean()) { mutatedOptions = options.entrySet() .stream() @@ -76,48 +82,48 @@ private static Map mutateOptions(Map options) { while (mutatedOptions.equals(options)) { if (randomBoolean()) { - mutatedOptions = mutateOption( - mutatedOptions, - KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), - () -> String.valueOf(randomBoolean()) - ); + mutatedOptions = mutateOption(mutatedOptions, CASE_INSENSITIVE_FIELD.getPreferredName(), () -> randomBoolean()); + } + + if (randomBoolean()) { + mutatedOptions = mutateOption(mutatedOptions, DEFAULT_FIELD_FIELD.getPreferredName(), () -> randomIdentifier()); } if (randomBoolean()) { - mutatedOptions = mutateOption( - mutatedOptions, - KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), - () -> randomIdentifier() - ); + mutatedOptions = mutateOption(mutatedOptions, TIME_ZONE_FIELD.getPreferredName(), () -> randomZone().getId()); } if (randomBoolean()) { - mutatedOptions = mutateOption( - mutatedOptions, - KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), - () -> randomZone().getId() - ); + mutatedOptions = mutateOption(mutatedOptions, "boost", () -> randomFloat() * 5.0f + 0.1f); } } return Collections.unmodifiableMap(mutatedOptions); } - private static Map mutateOption(Map options, String optionName, Supplier valueSupplier) { + private static Map mutateOption(Map options, String optionName, Supplier valueSupplier) { options = new HashMap<>(options); options.put(optionName, randomValueOtherThan(options.get(optionName), valueSupplier)); return options; } public void testQueryBuilding() { - KqlQueryBuilder qb = getBuilder(Map.of("case_insensitive", "false")); + KqlQueryBuilder qb = getBuilder(Map.of("case_insensitive", false)); assertThat(qb.caseInsensitive(), equalTo(false)); - qb = getBuilder(Map.of("case_insensitive", "false", "time_zone", "UTC", "default_field", "foo")); + qb = getBuilder(Map.of("case_insensitive", false, "time_zone", "UTC", "default_field", "foo")); assertThat(qb.caseInsensitive(), equalTo(false)); assertThat(qb.timeZone(), equalTo(ZoneId.of("UTC"))); assertThat(qb.defaultField(), equalTo("foo")); + qb = getBuilder(Map.of("boost", 2.5f)); + assertThat(qb.boost(), equalTo(2.5f)); + + qb = getBuilder(Map.of("case_insensitive", true, "boost", 1.5f, "default_field", "content")); + assertThat(qb.caseInsensitive(), equalTo(true)); + assertThat(qb.boost(), equalTo(1.5f)); + assertThat(qb.defaultField(), equalTo("content")); + Exception e = expectThrows(IllegalArgumentException.class, () -> getBuilder(Map.of("pizza", "yummy"))); assertThat(e.getMessage(), equalTo("illegal kql query option [pizza]")); @@ -125,7 +131,7 @@ public void testQueryBuilding() { assertThat(e.getMessage(), equalTo("Unknown time-zone ID: aoeu")); } - private static KqlQueryBuilder getBuilder(Map options) { + private static KqlQueryBuilder getBuilder(Map options) { final Source source = new Source(1, 1, StringUtils.EMPTY); final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", options); return (KqlQueryBuilder) kqlQuery.asBuilder(); @@ -136,4 +142,11 @@ public void testToString() { final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", Map.of()); assertEquals("KqlQuery@1:2[eggplant]", kqlQuery.toString()); } + + @Override + protected NamedWriteableRegistry writableRegistry() { + List entries = new ArrayList<>(ClusterModule.getNamedWriteables()); + entries.add(new NamedWriteableRegistry.Entry(KqlQueryBuilder.class, KqlQueryBuilder.NAME, KqlQueryBuilder::new)); + return new NamedWriteableRegistry(entries); + } } From 91be969cd2dbcfa43f8d257bbbcde1981e4b7042 Mon Sep 17 00:00:00 2001 From: afoucret Date: Thu, 2 Oct 2025 15:59:56 +0200 Subject: [PATCH 03/15] Improved documentation. --- .../expression/function/fulltext/Kql.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index b7a685dd175ce..3f3dc2361b540 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -20,7 +20,14 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.Foldables; -import org.elasticsearch.xpack.esql.expression.function.*; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.MapParam; +import org.elasticsearch.xpack.esql.expression.function.OptionalArgument; +import org.elasticsearch.xpack.esql.expression.function.Options; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; @@ -81,32 +88,34 @@ public Kql( ) Expression queryString, @MapParam( name = "options", - description = "(Optional) KQL additional options as <>.", + description = "(Optional) KQL additional options as <>." + + " See <> for more information.", params = { @MapParam.MapParamEntry( name = "case_insensitive", type = "boolean", valueHint = { "true", "false" }, - description = "If true, the query is case insensitive. Defaults to false." + description = "If true, performs case-insensitive matching for keyword fields. Defaults to false." ), @MapParam.MapParamEntry( name = "time_zone", type = "keyword", - valueHint = { "UTC", "America/New_York" }, - description = "Time zone to use when parsing date values. Defaults to UTC." + valueHint = {"UTC", "Europe/Paris", "America/New_York"}, + description = "UTC offset or IANA time zone used to interpret date literals in the query string." ), @MapParam.MapParamEntry( name = "default_field", type = "keyword", - valueHint = { "_all", "title" }, - description = "Default field to search when no field is specified in the query." + valueHint = {"*", "logs.*", "title"}, + description = "Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*)." ), @MapParam.MapParamEntry( name = "boost", type = "float", valueHint = { "2.5" }, description = "Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0." - ) }, + ) + }, optional = true ) Expression options ) { From b9b6022cb14a0d20249c44c743782b60cbc72d37 Mon Sep 17 00:00:00 2001 From: afoucret Date: Thu, 2 Oct 2025 16:33:12 +0200 Subject: [PATCH 04/15] Add verifier tests for KQL optional params. --- .../org/elasticsearch/xpack/esql/analysis/VerifierTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 9421951e2f691..ea5b6d29d1e54 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchPhrase; import org.elasticsearch.xpack.esql.expression.function.fulltext.MultiMatch; @@ -2295,6 +2296,9 @@ public void testFullTextFunctionOptions() { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", title, body, {\"%s\": %s})"); } + if (EsqlCapabilities.Cap.KQL_FUNCTION_OPTIONS.isEnabled()) { + checkOptionDataTypes(Kql.ALLOWED_OPTIONS, "FROM test | WHERE KQL(\"title: Jean\", {\"%s\": %s})"); + } } /** From cd416c0923f07b43e7737519cd5be1395b1a087f Mon Sep 17 00:00:00 2001 From: afoucret Date: Fri, 3 Oct 2025 08:54:50 +0200 Subject: [PATCH 05/15] KQL optional parameters CSV tests. --- .../esql/_snippets/functions/examples/kql.md | 3 +- .../functions/functionNamedParams/kql.md | 6 +- .../_snippets/functions/parameters/kql.md | 2 +- .../esql/kibana/definition/functions/kql.json | 10 +- .../src/main/resources/kql-function.csv-spec | 104 ++++++++++++++++++ .../xpack/esql/querydsl/query/KqlQuery.java | 5 + .../function/fulltext/KqlTests.java | 5 + 7 files changed, 125 insertions(+), 10 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md index 5679b9bca07c0..056a3529b71e8 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md @@ -16,7 +16,8 @@ FROM books | 3293 | Danny Faulkner | ```esql -null +FROM employees +| WHERE KQL("mary", {"case_insensitive": true, "default_field": "first_name", "boost": 1.5}) ``` diff --git a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md index bfa78e253feba..a458f6261dace 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md @@ -6,11 +6,11 @@ : (float) Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0. `time_zone` -: (keyword) Time zone to use when parsing date values. Defaults to UTC. +: (keyword) UTC offset or IANA time zone used to interpret date literals in the query string. `case_insensitive` -: (boolean) If true, the query is case insensitive. Defaults to false. +: (boolean) If true, performs case-insensitive matching for keyword fields. Defaults to false. `default_field` -: (keyword) Default field to search when no field is specified in the query. +: (keyword) Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*). diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md index 7e3c5fe2bec20..407600069fd75 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md @@ -6,5 +6,5 @@ : Query string in KQL query string format. `options` -: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). +: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). See [KQL query](/reference/query-languages/query-dsl/query-dsl-match-query.md#query-dsl-kql-query) for more information. diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json index 7493f5dc35499..9b1e9ce6f464b 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json @@ -27,9 +27,9 @@ { "name" : "options", "type" : "function_named_parameters", - "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, America/New_York], description='Time zone to use when parsing date values. Defaults to UTC.'}, {name='case_insensitive', values=[true, false], description='If true, the query is case insensitive. Defaults to false.'}, {name='default_field', values=[_all, title], description='Default field to search when no field is specified in the query.'}", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*).'}", "optional" : true, - "description" : "(Optional) KQL additional options as <>." + "description" : "(Optional) KQL additional options as <>. See <> for more information." } ], "variadic" : false, @@ -58,9 +58,9 @@ { "name" : "options", "type" : "function_named_parameters", - "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, America/New_York], description='Time zone to use when parsing date values. Defaults to UTC.'}, {name='case_insensitive', values=[true, false], description='If true, the query is case insensitive. Defaults to false.'}, {name='default_field', values=[_all, title], description='Default field to search when no field is specified in the query.'}", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*).'}", "optional" : true, - "description" : "(Optional) KQL additional options as <>." + "description" : "(Optional) KQL additional options as <>. See <> for more information." } ], "variadic" : false, @@ -69,7 +69,7 @@ ], "examples" : [ "FROM books\n| WHERE KQL(\"author: Faulkner\")", - null + "FROM employees\n| WHERE KQL(\"mary\", {\"case_insensitive\": true, \"default_field\": \"first_name\", \"boost\": 1.5})" ], "preview" : false, "snapshot_only" : false diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec index 8f7d3446c899c..7813dc6d32138 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -287,3 +287,107 @@ c: long | scalerank: long 10 | 3 15 | 2 ; + +kqlWithOptions +required_capability: kql_function +required_capability: kql_function_options + + +FROM employees +| WHERE KQL("first_name: Mary", {"case_insensitive": false}) +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; + +kqlWithCaseInsensitiveOption +required_capability: kql_function +required_capability: kql_function_options + +FROM employees +| WHERE KQL("first_name: mary", {"case_insensitive": true}) +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; + +kqlWithTimeZoneOption +required_capability: kql_function +required_capability: kql_function_options + +FROM logs +| WHERE KQL("@timestamp > \"2023-10-23T09:56:00\" AND @timestamp < \"2023-10-23T09:57:00\"", {"time_zone": "America/New_York"}) +| KEEP @timestamp, message +; + +@timestamp:date | message:text +2023-10-23T13:56:01.543Z | No response +2023-10-23T13:56:01.544Z | Running cats (cycle 2) +; + +kqlWithDefaultFieldOption +required_capability: kql_function +required_capability: kql_function_options + +FROM employees +| WHERE KQL("Support Engineer", {"default_field": "job_positions"}) +| KEEP emp_no, first_name, last_name, job_positions +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword +10004 | Chirstian | Koblick | [Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] +10015 | Guoxiang | Nooteboom | [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] +10021 | Ramzi | Erde | Support Engineer +; + +kqlWithBoostOption +required_capability: kql_function +required_capability: kql_function_options + +FROM employees METADATA _score +| WHERE KQL("job_positions: Support Engineer", {"boost": 2.5}) +| KEEP emp_no, first_name, last_name, job_positions, _score +| EVAL _score = round(_score, 2) +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double +10004 | Chirstian | Koblick | [Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] | 6.81 +10015 | Guoxiang | Nooteboom | [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] | 6.81 +10021 | Ramzi | Erde | Support Engineer | 6.81 +; + +kqlWithMultipleOptions +required_capability: kql_function +required_capability: kql_function_options +// tag::kql-with-options[] +FROM employees +| WHERE KQL("mary", {"case_insensitive": false, "default_field": "first_name", "boost": 1.5}) +// end::kql-with-options[] +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; + +kqlWithWildcardDefaultField +required_capability: kql_function +required_capability: kql_function_options + +x +FROM employees +| WHERE KQL("Mary", {"default_field": "*_name"}) +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java index 30908fe3d8cbe..07e8d370c876a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -66,6 +66,11 @@ public Map options() { return options; } + @Override + public boolean scorable() { + return true; + } + @Override protected String innerToString() { return query; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java index db2ada15ab49f..1b34aa9fefe31 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java @@ -90,4 +90,9 @@ protected Expression serializeDeserializeExpression(Expression expression) { // Fields use synthetic sources, which can't be serialized. So we use the originals instead. return newExpression.replaceChildren(expression.children()); } + + @Override + public void testFold() { + // kql query cannot be folded. + } } From 598cc3dde44b79f7ec53da02a20ced2b040968a7 Mon Sep 17 00:00:00 2001 From: afoucret Date: Fri, 3 Oct 2025 11:26:24 +0200 Subject: [PATCH 06/15] Update csv tests. --- .../qa/testFixtures/src/main/resources/kql-function.csv-spec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec index 7813dc6d32138..d72c9b299c306 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -369,7 +369,7 @@ required_capability: kql_function required_capability: kql_function_options // tag::kql-with-options[] FROM employees -| WHERE KQL("mary", {"case_insensitive": false, "default_field": "first_name", "boost": 1.5}) +| WHERE KQL("mary", {"case_insensitive": true, "default_field": "first_name", "boost": 1.5}) // end::kql-with-options[] | KEEP emp_no, first_name, last_name ; @@ -382,7 +382,6 @@ kqlWithWildcardDefaultField required_capability: kql_function required_capability: kql_function_options -x FROM employees | WHERE KQL("Mary", {"default_field": "*_name"}) | KEEP emp_no, first_name, last_name From 1161831c9109e4b275d5434d3458f5f578e15dea Mon Sep 17 00:00:00 2001 From: afoucret Date: Fri, 3 Oct 2025 11:26:39 +0200 Subject: [PATCH 07/15] Lint. --- .../esql/_snippets/functions/functionNamedParams/kql.md | 2 +- .../esql/kibana/definition/functions/kql.json | 4 ++-- .../xpack/esql/expression/function/fulltext/Kql.java | 5 ++--- .../xpack/esql/querydsl/query/KqlQueryTests.java | 7 ++++++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md index a458f6261dace..b0dbba48647b2 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md @@ -12,5 +12,5 @@ : (boolean) If true, performs case-insensitive matching for keyword fields. Defaults to false. `default_field` -: (keyword) Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*). +: (keyword) Default field to search if no field is provided in the query string. Supports wildcards (*). diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json index 9b1e9ce6f464b..19748580c2bfb 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json @@ -27,7 +27,7 @@ { "name" : "options", "type" : "function_named_parameters", - "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*).'}", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", "optional" : true, "description" : "(Optional) KQL additional options as <>. See <> for more information." } @@ -58,7 +58,7 @@ { "name" : "options", "type" : "function_named_parameters", - "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*).'}", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", "optional" : true, "description" : "(Optional) KQL additional options as <>. See <> for more information." } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index 3f3dc2361b540..f58b3b44bd2de 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -107,15 +107,14 @@ public Kql( name = "default_field", type = "keyword", valueHint = {"*", "logs.*", "title"}, - description = "Default field (or field pattern with wildcards) to target when a bare term in the query does not specify a field. Supports wildcards (*)." + description = "Default field to search if no field is provided in the query string. Supports wildcards (*)." ), @MapParam.MapParamEntry( name = "boost", type = "float", valueHint = { "2.5" }, description = "Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0." - ) - }, + )}, optional = true ) Expression options ) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java index 0ea107b1ef7e8..fd82c11b60085 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java @@ -16,7 +16,12 @@ import java.time.ZoneId; import java.time.zone.ZoneRulesException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; From 6225fda3fff6e76aaa4a53f9e41e5f36726bc6ed Mon Sep 17 00:00:00 2001 From: afoucret Date: Fri, 3 Oct 2025 11:27:53 +0200 Subject: [PATCH 08/15] KQL query now support non-case sensitive searches when no field is specified. --- .../xpack/kql/parser/KqlAstBuilder.java | 87 +++++--- .../xpack/kql/parser/KqlParsingContext.java | 32 ++- .../parser/KqlParserFieldlessQueryTests.java | 191 ++++++++++++++++++ 3 files changed, 280 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java index 9d7aeb4bf0ddf..64e158c6c9261 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java @@ -26,6 +26,8 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isDateField; @@ -181,15 +183,32 @@ public QueryBuilder visitRangeQuery(KqlBaseParser.RangeQueryContext ctx) { @Override public QueryBuilder visitFieldLessQuery(KqlBaseParser.FieldLessQueryContext ctx) { String queryText = extractText(ctx.fieldQueryValue()); + boolean hasWildcard = hasWildcard(ctx.fieldQueryValue()); + boolean isPhraseMatch = ctx.fieldQueryValue().fieldQueryValueLiteral().QUOTED_STRING() != null; + + if (kqlParsingContext.caseInsensitive() && isPhraseMatch == false) { + // Special handling for case-insenitive queries. + // We can't use a query_string or a match query since it does not support case-insensitive on all field types. + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + Set searchFields = kqlParsingContext.resolveDefaultFieldNames() + .stream() + .filter(kqlParsingContext::isSearchableField) + .filter(Predicate.not(kqlParsingContext::isDateField)) // Date fields do not support case insensitive + .filter(Predicate.not(kqlParsingContext::isRangeField)) // Range fields do not support case insensitive + .collect(Collectors.toSet()); + + withFields(searchFields, fieldQueryApplier(queryText, hasWildcard, false, boolQueryBuilder::should)); + return rewriteDisjunctionQuery(boolQueryBuilder); + } - if (hasWildcard(ctx.fieldQueryValue())) { + if (hasWildcard) { + // Wildcard queries are using a query_string query QueryStringQueryBuilder queryString = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)); if (kqlParsingContext.defaultField() != null) { queryString.defaultField(kqlParsingContext.defaultField()); } return queryString; } - boolean isPhraseMatch = ctx.fieldQueryValue().fieldQueryValueLiteral().QUOTED_STRING() != null; MultiMatchQueryBuilder multiMatchQuery = QueryBuilders.multiMatchQuery(queryText) .type(isPhraseMatch ? MultiMatchQueryBuilder.Type.PHRASE : MultiMatchQueryBuilder.Type.BEST_FIELDS) @@ -250,37 +269,47 @@ public QueryBuilder parseFieldQuery( BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); String queryText = extractText(fieldQueryValueCtx.fieldQueryValueLiteral()); boolean hasWildcard = hasWildcard(fieldQueryValueCtx); + boolean isPhraseMatch = fieldQueryValueCtx.fieldQueryValueLiteral().QUOTED_STRING() != null; - withFields(fieldNameCtx, (fieldName, mappedFieldType) -> { - QueryBuilder fieldQuery; - - if (hasWildcard && isKeywordField(mappedFieldType)) { - fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); - } else if (hasWildcard) { - fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName); - } else if (isDateField(mappedFieldType)) { - RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText); - if (kqlParsingContext.timeZone() != null) { - rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId()); - } - fieldQuery = rangeFieldQuery; - } else if (isKeywordField(mappedFieldType)) { - fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); - } else if (fieldQueryValueCtx.fieldQueryValueLiteral().QUOTED_STRING() != null) { - fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText); - } else { - fieldQuery = QueryBuilders.matchQuery(fieldName, queryText); - } - - if (fieldQuery != null) { - boolQueryBuilder.should(wrapWithNestedQuery(fieldName, fieldQuery)); - } - }); + withFields(fieldNameCtx, fieldQueryApplier(queryText, hasWildcard, isPhraseMatch, boolQueryBuilder::should)); return rewriteDisjunctionQuery(boolQueryBuilder); } } + private BiConsumer fieldQueryApplier( + String queryText, + boolean hasWildcard, + boolean isPhraseMatch, + Consumer clauseConsumer + ) { + return (fieldName, mappedFieldType) -> { + QueryBuilder fieldQuery; + + if (hasWildcard && isKeywordField(mappedFieldType)) { + fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); + } else if (hasWildcard) { + fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName); + } else if (isDateField(mappedFieldType)) { + RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText); + if (kqlParsingContext.timeZone() != null) { + rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId()); + } + fieldQuery = rangeFieldQuery; + } else if (isKeywordField(mappedFieldType)) { + fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); + } else if (isPhraseMatch) { + fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText); + } else { + fieldQuery = QueryBuilders.matchQuery(fieldName, queryText).lenient(true); + } + + if (fieldQuery != null) { + clauseConsumer.accept(wrapWithNestedQuery(fieldName, fieldQuery)); + } + }; + } + private static boolean isAndQuery(ParserRuleContext ctx) { return switch (ctx) { case KqlBaseParser.BooleanQueryContext booleanQueryCtx -> booleanQueryCtx.operator.getType() == KqlBaseParser.AND; @@ -311,6 +340,10 @@ private void withFields(KqlBaseParser.FieldNameContext ctx, BiConsumer fieldNames, BiConsumer fieldConsummer) { fieldNames.forEach(fieldName -> { MappedFieldType fieldType = kqlParsingContext.fieldType(fieldName); if (isSearchableField(fieldName, fieldType)) { diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java index 30740833ee40e..449ea740a5659 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java @@ -13,10 +13,12 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.NestedObjectMapper; +import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.support.NestedScope; import java.time.ZoneId; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -78,7 +80,22 @@ public Set resolveFieldNames(String fieldNamePattern) { } public Set resolveDefaultFieldNames() { - return resolveFieldNames(defaultField); + if (defaultField != null) { + return resolveFieldNames(defaultField); + } + + assert queryRewriteContext.getIndexSettings() != null; + + if (queryRewriteContext.getIndexSettings().getDefaultFields().isEmpty()) { + return resolveFieldNames("*"); + } + + Set fieldNames = new HashSet<>(); + queryRewriteContext.getIndexSettings().getDefaultFields().forEach(fieldNamePattern -> { + fieldNames.addAll(resolveFieldNames(fieldNamePattern)); + }); + + return fieldNames; } public MappedFieldType fieldType(String fieldName) { @@ -89,8 +106,17 @@ public static boolean isRuntimeField(MappedFieldType fieldType) { return fieldType instanceof AbstractScriptFieldType; } + public boolean isDateField(String fieldName) { + return isDateField(fieldType(fieldName)); + } + + public boolean isRangeField(String fieldName) { + return fieldType(fieldName) != null && fieldType(fieldName) instanceof RangeFieldMapper.RangeFieldType; + } + public static boolean isDateField(MappedFieldType fieldType) { - return fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE); + return fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE) + || fieldType.typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE); } public static boolean isKeywordField(MappedFieldType fieldType) { @@ -139,7 +165,7 @@ private Map nestedMappers() { public static class Builder { private final QueryRewriteContext queryRewriteContext; - private boolean caseInsensitive = true; + private boolean caseInsensitive = false; private ZoneId timeZone = null; private String defaultField = null; diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java index c1f080fdc1eb4..86d5a407eb099 100644 --- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java @@ -7,7 +7,20 @@ package org.elasticsearch.xpack.kql.parser; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; public class KqlParserFieldlessQueryTests extends AbstractKqlParserTestCase { @@ -92,4 +105,182 @@ public void testParseQuotedStringQuery() { // Containing unescaped KQL reserved characters assertMultiMatchQuery(parseKqlQuery("\"foo*: {(})\""), "foo*: {(})", MultiMatchQueryBuilder.Type.PHRASE); } + + public void testParseCaseInsensitiveFieldlessQueries() { + // Test case-insensitive unquoted literal queries + QueryBuilder query = parseKqlQueryCaseInsensitive("Foo"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify that each should clause targets a searchable non-date field + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("Foo")); + assertThat(matchQuery.lenient(), equalTo(true)); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("Foo")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + } + } + + public void testParseCaseInsensitiveWildcardQueries() { + // Test case-insensitive wildcard queries + QueryBuilder query = parseKqlQueryCaseInsensitive("Fo*"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify that each should clause handles wildcards appropriately + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof WildcardQueryBuilder wildcardQuery) { + assertThat(wildcardQuery.value(), equalTo("Fo*")); + assertThat(wildcardQuery.caseInsensitive(), equalTo(true)); + } else if (clause instanceof QueryStringQueryBuilder queryStringQuery) { + assertThat(queryStringQuery.queryString(), equalTo("Fo*")); + } + } + } + + public void testParseCaseInsensitiveQuotedStringQueries() { + // Test case-insensitive phrase queries + // Note: With the current implementation, phrase matches use standard MultiMatchQuery + // even in case-insensitive mode (they bypass the special case-insensitive handling) + QueryBuilder query = parseKqlQueryCaseInsensitive("\"Foo Bar\""); + assertThat(query, instanceOf(MultiMatchQueryBuilder.class)); + + MultiMatchQueryBuilder multiMatchQuery = (MultiMatchQueryBuilder) query; + assertThat(multiMatchQuery.value(), equalTo("Foo Bar")); + assertThat(multiMatchQuery.type(), equalTo(MultiMatchQueryBuilder.Type.PHRASE)); + assertThat(multiMatchQuery.lenient(), equalTo(true)); + } + + public void testParseCaseInsensitiveMultipleWords() { + // Test case-insensitive multiple word queries + QueryBuilder query = parseKqlQueryCaseInsensitive("Foo Bar Baz"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify that each should clause is a match query with multiple words + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("Foo Bar Baz")); + assertThat(matchQuery.lenient(), equalTo(true)); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("Foo Bar Baz")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + } + } + + public void testParseCaseInsensitiveWildcardWithSpecialChars() { + // Test case-insensitive wildcard queries with special characters that need escaping + QueryBuilder query = parseKqlQueryCaseInsensitive("Fo*[bar]"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify proper escaping in query string queries + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof QueryStringQueryBuilder queryStringQuery) { + assertThat(queryStringQuery.queryString(), equalTo("Fo*\\[bar\\]")); + } + } + } + + public void testCaseInsensitiveWithEscapedCharacters() { + // Test case-insensitive queries with escaped characters + QueryBuilder query = parseKqlQueryCaseInsensitive("Foo\\*Bar"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // Should not be treated as wildcard since asterisk is escaped + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("Foo*Bar")); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("Foo*Bar")); + } + } + } + + public void testCaseInsensitiveWithCustomDefaultField() { + // Test case-insensitive queries with a custom default field pattern + QueryBuilder query = parseKqlQueryCaseInsensitiveWithDefaultField("foo", KEYWORD_FIELD_NAME); + + assertThat(query, instanceOf(TermQueryBuilder.class)); + TermQueryBuilder termQuery = (TermQueryBuilder) query; + assertThat(termQuery.fieldName(), equalTo(KEYWORD_FIELD_NAME)); + assertThat(termQuery.value(), equalTo("foo")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + + public void testCaseInsensitiveWithWildcardDefaultField() { + // Test case-insensitive queries with a wildcard default field pattern + QueryBuilder query = parseKqlQueryCaseInsensitiveWithDefaultField("foo", "mapped_*"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // Should target all mapped fields that match the pattern (excluding date fields) + List expectedFields = searchableFields("mapped_*").stream() + .filter(fieldName -> fieldName.contains("date") == false && fieldName.contains("range") == false) + .toList(); + assertThat(boolQuery.should(), hasSize(expectedFields.size())); + + // Verify that all should clauses target appropriate fields + List shouldClauses = boolQuery.should(); // Verify that each should clause is properly configured + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("foo")); + assertThat(matchQuery.lenient(), equalTo(true)); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("foo")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + } + } + + public void testCaseInsensitiveEmptyResultHandling() { + // Test behavior when no fields match after filtering (edge case) + // This creates a scenario where all default fields are date fields + QueryBuilder query = parseKqlQueryCaseInsensitiveWithDefaultField("test", DATE_FIELD_NAME); + assertThat(query, instanceOf(MatchNoneQueryBuilder.class)); + } + + /** + * Helper method to parse KQL query with case-insensitive mode enabled + */ + private QueryBuilder parseKqlQueryCaseInsensitive(String kqlQuery) { + KqlParser parser = new KqlParser(); + KqlParsingContext kqlParserContext = KqlParsingContext.builder(createQueryRewriteContext()).caseInsensitive(true).build(); + return parser.parseKqlQuery(kqlQuery, kqlParserContext); + } + + /** + * Helper method to parse KQL query with case-insensitive mode and custom default field + */ + private QueryBuilder parseKqlQueryCaseInsensitiveWithDefaultField(String kqlQuery, String defaultField) { + KqlParser parser = new KqlParser(); + KqlParsingContext kqlParserContext = KqlParsingContext.builder(createQueryRewriteContext()) + .caseInsensitive(true) + .defaultField(defaultField) + .build(); + return parser.parseKqlQuery(kqlQuery, kqlParserContext); + } + + /** + * Helper method to count the number of excluded fields in searchable fields. + * This is used to calculate expected should clause counts since date and range fields + * are filtered out in case-insensitive queries. + */ + private int excludedFieldCount() { + return (int) searchableFields().stream().filter(fieldName -> fieldName.contains("date") || fieldName.contains("range")).count(); + } } From bb07a50b09bd186a1509443087e39f0f9c658cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= Date: Fri, 3 Oct 2025 11:44:38 +0200 Subject: [PATCH 09/15] Update docs/changelog/135895.yaml --- docs/changelog/135895.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/135895.yaml diff --git a/docs/changelog/135895.yaml b/docs/changelog/135895.yaml new file mode 100644 index 0000000000000..a8f660ee815c9 --- /dev/null +++ b/docs/changelog/135895.yaml @@ -0,0 +1,6 @@ +pr: 135895 +summary: Add optional parameters support to KQL function +area: ES|QL +type: enhancement +issues: + - 135823 From 55c215a7c4ad0318c31288102c80db8f6e0134d8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 3 Oct 2025 09:55:55 +0000 Subject: [PATCH 10/15] [CI] Auto commit changes from spotless --- .../xpack/esql/expression/function/fulltext/Kql.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index f58b3b44bd2de..3c6bbcff33a51 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -100,13 +100,13 @@ public Kql( @MapParam.MapParamEntry( name = "time_zone", type = "keyword", - valueHint = {"UTC", "Europe/Paris", "America/New_York"}, + valueHint = { "UTC", "Europe/Paris", "America/New_York" }, description = "UTC offset or IANA time zone used to interpret date literals in the query string." ), @MapParam.MapParamEntry( name = "default_field", type = "keyword", - valueHint = {"*", "logs.*", "title"}, + valueHint = { "*", "logs.*", "title" }, description = "Default field to search if no field is provided in the query string. Supports wildcards (*)." ), @MapParam.MapParamEntry( @@ -114,7 +114,7 @@ public Kql( type = "float", valueHint = { "2.5" }, description = "Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0." - )}, + ) }, optional = true ) Expression options ) { From 5f872b3705dea1d43cb48b88f9e7c2ddce0dbd61 Mon Sep 17 00:00:00 2001 From: afoucret Date: Fri, 3 Oct 2025 12:45:31 +0200 Subject: [PATCH 11/15] Doc improvements. --- .../esql/_snippets/functions/examples/kql.md | 4 ++++ .../esql/_snippets/functions/parameters/kql.md | 2 +- .../esql/kibana/definition/functions/kql.json | 4 ++-- .../xpack/esql/expression/function/fulltext/Kql.java | 11 +++++++++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md index 056a3529b71e8..f5fdd98d16547 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md @@ -15,6 +15,10 @@ FROM books | 2883 | William Faulkner | | 3293 | Danny Faulkner | +```{applies_to} +stack: 9.3.0 +``` + ```esql FROM employees | WHERE KQL("mary", {"case_insensitive": true, "default_field": "first_name", "boost": 1.5}) diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md index 407600069fd75..7e3c5fe2bec20 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md @@ -6,5 +6,5 @@ : Query string in KQL query string format. `options` -: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). See [KQL query](/reference/query-languages/query-dsl/query-dsl-match-query.md#query-dsl-kql-query) for more information. +: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json index 19748580c2bfb..096efc65ed312 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json @@ -29,7 +29,7 @@ "type" : "function_named_parameters", "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", "optional" : true, - "description" : "(Optional) KQL additional options as <>. See <> for more information." + "description" : "(Optional) KQL additional options as <>." } ], "variadic" : false, @@ -60,7 +60,7 @@ "type" : "function_named_parameters", "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", "optional" : true, - "description" : "(Optional) KQL additional options as <>. See <> for more information." + "description" : "(Optional) KQL additional options as <>." } ], "variadic" : false, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index 3c6bbcff33a51..23576f30a3590 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -77,7 +77,14 @@ public class Kql extends FullTextFunction implements OptionalArgument { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.0.0"), @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.GA, version = "9.1.0") }, description = "Performs a KQL query. Returns true if the provided KQL query string matches the row.", - examples = { @Example(file = "kql-function", tag = "kql-with-field"), @Example(file = "kql-function", tag = "kql-with-options") } + examples = { + @Example(file = "kql-function", tag = "kql-with-field", description = "Use KQL to filter by a specific field value"), + @Example( + file = "kql-function", + tag = "kql-with-options", + description = "Use KQL with additional options for case-insensitive matching and custom settings", + applies_to = "stack: 9.3.0" + ) } ) public Kql( Source source, @@ -89,7 +96,7 @@ public Kql( @MapParam( name = "options", description = "(Optional) KQL additional options as <>." - + " See <> for more information.", + + " Available in stack version 9.3.0 and later.", params = { @MapParam.MapParamEntry( name = "case_insensitive", From 414b680faaa5fbfb9c8b4dbdf9eb8876fe6a8aeb Mon Sep 17 00:00:00 2001 From: afoucret Date: Fri, 3 Oct 2025 12:50:47 +0200 Subject: [PATCH 12/15] Another doc fix. --- .../esql/_snippets/functions/examples/kql.md | 6 +++++- .../esql/_snippets/functions/parameters/kql.md | 2 +- .../esql/kibana/definition/functions/kql.json | 4 ++-- .../xpack/esql/expression/function/fulltext/Kql.java | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md index f5fdd98d16547..2d0ad3d84c2d5 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md @@ -2,6 +2,8 @@ **Examples** +Use KQL to filter by a specific field value + ```esql FROM books | WHERE KQL("author: Faulkner") @@ -16,9 +18,11 @@ FROM books | 3293 | Danny Faulkner | ```{applies_to} -stack: 9.3.0 +stack: ga 9.3.0 ``` +Use KQL with additional options for case-insensitive matching and custom settings + ```esql FROM employees | WHERE KQL("mary", {"case_insensitive": true, "default_field": "first_name", "boost": 1.5}) diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md index 7e3c5fe2bec20..9ffaec37b1f56 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md @@ -6,5 +6,5 @@ : Query string in KQL query string format. `options` -: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). +: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). Available in stack version 9.3.0 and later. diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json index 096efc65ed312..761e462f1b53e 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json @@ -29,7 +29,7 @@ "type" : "function_named_parameters", "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", "optional" : true, - "description" : "(Optional) KQL additional options as <>." + "description" : "(Optional) KQL additional options as <>. Available in stack version 9.3.0 and later." } ], "variadic" : false, @@ -60,7 +60,7 @@ "type" : "function_named_parameters", "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", "optional" : true, - "description" : "(Optional) KQL additional options as <>." + "description" : "(Optional) KQL additional options as <>. Available in stack version 9.3.0 and later." } ], "variadic" : false, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index 23576f30a3590..3f0eac393e7fe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -83,7 +83,7 @@ public class Kql extends FullTextFunction implements OptionalArgument { file = "kql-function", tag = "kql-with-options", description = "Use KQL with additional options for case-insensitive matching and custom settings", - applies_to = "stack: 9.3.0" + applies_to = "stack: ga 9.3.0" ) } ) public Kql( From bc2ceb12b0c3e3bc2430bebff7985af3295672cb Mon Sep 17 00:00:00 2001 From: afoucret Date: Fri, 3 Oct 2025 19:13:57 +0200 Subject: [PATCH 13/15] Restore content removed by mistake. --- .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 024cd472f936e..acabb98d843b2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -900,6 +900,11 @@ public enum Cap { /** * Full text functions can be used in disjunctions */ + FULL_TEXT_FUNCTIONS_DISJUNCTIONS, + + /** + * Change field caps response for semantic_text fields to be reported as text + */ SEMANTIC_TEXT_FIELD_CAPS, /** From d16422eda4ee7e2d7c055012de993529caf3a6ee Mon Sep 17 00:00:00 2001 From: afoucret Date: Mon, 6 Oct 2025 10:33:48 +0200 Subject: [PATCH 14/15] Fix test errors. --- .../rest-api-spec/test/kql/20_kql_match_query.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml index 8255cd147d950..4f6e1d0ae64e5 100644 --- a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml +++ b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml @@ -118,7 +118,6 @@ setup: --- "KQL match term queries (integer field)": - do: - catch: bad_request search: index: test-index rest_total_hits_as_int: true @@ -126,10 +125,7 @@ setup: { "query": { "kql": { "query": "integer_field: foo" } } } - - match: { error.type: "search_phase_execution_exception" } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } - - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + - match: { hits.total: 0 } - do: search: @@ -157,7 +153,6 @@ setup: --- "KQL match term queries (double field)": - do: - catch: bad_request search: index: test-index rest_total_hits_as_int: true @@ -165,10 +160,7 @@ setup: { "query": { "kql": { "query": "double_field: foo" } } } - - match: { error.type: "search_phase_execution_exception" } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } - - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + - match: { hits.total: 0 } - do: search: From 5bcfc58b8b113bbe5d251852aa3c4eadb8a65759 Mon Sep 17 00:00:00 2001 From: afoucret Date: Tue, 7 Oct 2025 08:53:09 +0200 Subject: [PATCH 15/15] Fixing flakiness in CSV tests. --- .../src/main/resources/kql-function.csv-spec | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec index d72c9b299c306..204d17f57ab50 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -322,6 +322,7 @@ required_capability: kql_function_options FROM logs | WHERE KQL("@timestamp > \"2023-10-23T09:56:00\" AND @timestamp < \"2023-10-23T09:57:00\"", {"time_zone": "America/New_York"}) | KEEP @timestamp, message +| SORT @timestamp ASC ; @timestamp:date | message:text @@ -352,16 +353,15 @@ required_capability: kql_function_options FROM employees METADATA _score | WHERE KQL("job_positions: Support Engineer", {"boost": 2.5}) -| KEEP emp_no, first_name, last_name, job_positions, _score -| EVAL _score = round(_score, 2) +| KEEP emp_no, first_name, last_name, job_positions | SORT emp_no | LIMIT 3 ; -emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double -10004 | Chirstian | Koblick | [Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] | 6.81 -10015 | Guoxiang | Nooteboom | [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] | 6.81 -10021 | Ramzi | Erde | Support Engineer | 6.81 +emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword +10004 | Chirstian | Koblick | [Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] +10015 | Guoxiang | Nooteboom | [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] +10021 | Ramzi | Erde | Support Engineer ; kqlWithMultipleOptions