diff --git a/docs/reference/esql/functions/functionNamedParams/qstr.asciidoc b/docs/reference/esql/functions/functionNamedParams/qstr.asciidoc deleted file mode 100644 index e12257c0bc2a0..0000000000000 --- a/docs/reference/esql/functions/functionNamedParams/qstr.asciidoc +++ /dev/null @@ -1,29 +0,0 @@ -// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. - -*Supported function named parameters* - -[%header.monospaced.styled,format=dsv,separator=|] -|=== -name | types | description -max_determinized_states | [integer] | Maximum number of automaton states required for the query. Default is 10000. -fuzziness | [keyword] | Maximum edit distance allowed for matching. -auto_generate_synonyms_phrase_query | [boolean] | If true, match phrase queries are automatically created for multi-term synonyms. -phrase_slop | [integer] | Maximum number of positions allowed between matching tokens for phrases. -default_field | [keyword] | Default field to search if no field is provided in the query string. Supports wildcards (*). -allow_leading_wildcard | [boolean] | If true, the wildcard characters * and ? are allowed as the first character of the query string. -minimum_should_match | [string] | Minimum number of clauses that must match for a document to be returned. -fuzzy_transpositions | [boolean] | If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). -fuzzy_prefix_length | [integer] | Number of beginning characters left unchanged for fuzzy matching. Defaults to 0. -time_zone | [keyword] | Coordinated Universal Time (UTC) offset or IANA time zone used to convert date values in the query string to UTC. -lenient | [boolean] | If false, format-based errors, such as providing a text query value for a numeric field, are returned. -rewrite | [keyword] | Method used to rewrite the query. -default_operator | [keyword] | Default boolean logic used to interpret text in the query string if no operators are specified. -analyzer | [keyword] | Analyzer used to convert the text in the query value into token. -fuzzy_max_expansions | [integer] | Maximum number of terms to which the query expands for fuzzy matching. Defaults to 50. -quote_analyzer | [keyword] | Analyzer used to convert quoted text in the query string into tokens. -allow_wildcard | [boolean] | If true, the query attempts to analyze wildcard terms in the query string. -boost | [float] | Floating point number used to decrease or increase the relevance scores of the query. -quote_field_suffix | [keyword] | Suffix appended to quoted text in the query string. -enable_position_increments | [boolean] | If true, enable position increments in queries constructed from a query_string search. Defaults to true. -fields | [keyword] | Array of fields to search. Supports wildcards (*). -|=== diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/dissect.json b/docs/reference/query-languages/esql/kibana/definition/commands/dissect.json new file mode 100644 index 0000000000000..79619eafddd92 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/dissect.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "dissect" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/drop.json b/docs/reference/query-languages/esql/kibana/definition/commands/drop.json new file mode 100644 index 0000000000000..3b735ea950673 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/drop.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "drop" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/enrich.json b/docs/reference/query-languages/esql/kibana/definition/commands/enrich.json new file mode 100644 index 0000000000000..0cd2d14e58928 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/enrich.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "enrich" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/eval.json b/docs/reference/query-languages/esql/kibana/definition/commands/eval.json new file mode 100644 index 0000000000000..eceee558d96a8 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/eval.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "eval" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/explain.json b/docs/reference/query-languages/esql/kibana/definition/commands/explain.json new file mode 100644 index 0000000000000..8f626e52f53de --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/explain.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "explain" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/grok.json b/docs/reference/query-languages/esql/kibana/definition/commands/grok.json new file mode 100644 index 0000000000000..2c1ec9127d3e5 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/grok.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "grok" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/inlinestats.json b/docs/reference/query-languages/esql/kibana/definition/commands/inlinestats.json new file mode 100644 index 0000000000000..081220fcebc86 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/inlinestats.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "inlinestats" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/insist.json b/docs/reference/query-languages/esql/kibana/definition/commands/insist.json new file mode 100644 index 0000000000000..4629a141faa70 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/insist.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "insist" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/keep.json b/docs/reference/query-languages/esql/kibana/definition/commands/keep.json new file mode 100644 index 0000000000000..9a4a46d78ae29 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/keep.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "keep" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/limit.json b/docs/reference/query-languages/esql/kibana/definition/commands/limit.json new file mode 100644 index 0000000000000..74011a821278b --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/limit.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "limit" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/lookup.json b/docs/reference/query-languages/esql/kibana/definition/commands/lookup.json new file mode 100644 index 0000000000000..2e1aada587943 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/lookup.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "lookup" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/lookup_join.json b/docs/reference/query-languages/esql/kibana/definition/commands/lookup_join.json new file mode 100644 index 0000000000000..5d7ef204afde6 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/lookup_join.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "lookup_join" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/mv_expand.json b/docs/reference/query-languages/esql/kibana/definition/commands/mv_expand.json new file mode 100644 index 0000000000000..e84f5dd162d32 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/mv_expand.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "mv_expand" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/rename.json b/docs/reference/query-languages/esql/kibana/definition/commands/rename.json new file mode 100644 index 0000000000000..1e5747a06d32b --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/rename.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "rename" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/sort.json b/docs/reference/query-languages/esql/kibana/definition/commands/sort.json new file mode 100644 index 0000000000000..f40ad7863ddb4 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/sort.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "sort" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/stats.json b/docs/reference/query-languages/esql/kibana/definition/commands/stats.json new file mode 100644 index 0000000000000..46bec717cebd9 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/stats.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "stats" +} diff --git a/docs/reference/query-languages/esql/kibana/definition/commands/where.json b/docs/reference/query-languages/esql/kibana/definition/commands/where.json new file mode 100644 index 0000000000000..2e93beb94b489 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/commands/where.json @@ -0,0 +1,5 @@ +{ + "comment" : "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "command", + "name" : "where" +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 8f15dc5b7d9d3..958ac84ac82bb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.core.PathUtils; import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.xcontent.XContentBuilder; @@ -39,6 +40,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.session.Configuration; import java.io.BufferedReader; @@ -874,6 +876,47 @@ void renderDetailedDescription(String detailedDescription, String note) throws I } } + /** Command specific docs generating, currently very empty since we only render kibana definition files */ + public static class CommandsDocsSupport extends DocsV3Support { + private final LogicalPlan command; + private final XPackLicenseState licenseState; + + public CommandsDocsSupport(String name, Class testClass, LogicalPlan command, XPackLicenseState licenseState) { + super("commands", name, testClass, Map::of); + this.command = command; + this.licenseState = licenseState; + } + + @Override + public void renderSignature() throws IOException { + // Unimplemented until we make command docs dynamically generated + } + + @Override + public void renderDocs() throws Exception { + // Currently we only render kibana definition files, but we could expand to rendering much more if we decide to + renderKibanaCommandDefinition(); + } + + void renderKibanaCommandDefinition() throws Exception { + try (XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint().lfAtEnd().startObject()) { + builder.field( + "comment", + "This is generated by ESQL’s DocsV3Support. Do not edit it. See ../README.md for how to regenerate it." + ); + builder.field("type", "command"); + builder.field("name", name); + License.OperationMode license = licenseState.getOperationMode(); + if (license != null && license != License.OperationMode.BASIC) { + builder.field("license", license.toString()); + } + String rendered = Strings.toString(builder.endObject()); + logger.info("Writing kibana command definition for [{}]:\n{}", name, rendered); + writeToTempKibanaDir("definition", "json", rendered); + } + } + } + protected String buildFunctionSignatureSvg() throws IOException { return (definition != null) ? RailRoadDiagram.functionSignature(definition) : null; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/CommandLicenseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/CommandLicenseTests.java new file mode 100644 index 0000000000000..55f691842236a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/CommandLicenseTests.java @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.LicenseAware; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; +import org.elasticsearch.xpack.esql.parser.EsqlBaseParserVisitor; +import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class CommandLicenseTests extends ESTestCase { + private static final Logger log = LogManager.getLogger(CommandLicenseTests.class); + + public void testLicenseCheck() { + var sourceCommand = new LocalRelation(Source.EMPTY, List.of(), null); + for (var commandName : getCommandClasses().keySet()) { + Class commandClass = getCommandClasses().get(commandName); + try { + checkLicense(commandName, createInstance(commandClass, sourceCommand)); + } catch (Exception e) { + Throwable c = e.getCause(); + fail("Failed to create instance of command class: " + commandClass.getName() + " - " + e.getMessage() + " - " + c); + } + } + } + + public static class TestCheckLicense { + XPackLicenseState basicLicense = makeLicenseState(License.OperationMode.BASIC); + XPackLicenseState platinumLicense = makeLicenseState(License.OperationMode.PLATINUM); + XPackLicenseState enterpriseLicense = makeLicenseState(License.OperationMode.ENTERPRISE); + + private XPackLicenseState licenseLevel(LicenseAware licenseAware) { + for (XPackLicenseState license : List.of(basicLicense, platinumLicense, enterpriseLicense)) { + if (licenseAware.licenseCheck(license)) { + return license; + } + } + throw new IllegalArgumentException("No license level is supported by " + licenseAware.getClass().getName()); + } + } + + private static XPackLicenseState makeLicenseState(License.OperationMode mode) { + return new XPackLicenseState(System::currentTimeMillis, new XPackLicenseStatus(mode, true, null)); + } + + private static void checkLicense(String commandName, LogicalPlan command) throws Exception { + log.info("Running function license checks"); + TestCheckLicense checkLicense = new TestCheckLicense(); + if (command instanceof LicenseAware licenseAware) { + log.info("Command " + commandName + " implements LicenseAware."); + saveLicenseState(commandName, command, checkLicense.licenseLevel(licenseAware)); + } else { + log.info("Command " + commandName + " does not implement LicenseAware."); + saveLicenseState(commandName, command, checkLicense.basicLicense); + } + } + + private static void saveLicenseState(String name, LogicalPlan command, XPackLicenseState licenseState) throws Exception { + DocsV3Support.CommandsDocsSupport docs = new DocsV3Support.CommandsDocsSupport( + name.toLowerCase(Locale.ROOT), + CommandLicenseTests.class, + command, + licenseState + ); + docs.renderDocs(); + } + + // Find all command classes, by looking at the public methods of the EsqlBaseParserVisitor + private static Map> getCommandClasses() { + Map> commandClasses = new TreeMap<>(); + Pattern pattern = Pattern.compile("visit(\\w+)Command"); + String planPackage = "org.elasticsearch.xpack.esql.plan.logical"; + Map commandClassNameMapper = Map.of( + "Where", + "Filter", + "Inlinestats", + "InlineStats", + "Rrf", + "RrfScoreEval", + "Sort", + "OrderBy", + "Stats", + "Aggregate", + "Join", + "LookupJoin" + ); + Map commandNameMapper = Map.of("ChangePoint", "CHANGE_POINT", "LookupJoin", "LOOKUP_JOIN", "MvExpand", "MV_EXPAND"); + Map commandPackageMapper = Map.of("Rerank", planPackage + ".inference", "LookupJoin", planPackage + ".join"); + Set ignoredClasses = Set.of("Processing", "TimeSeries", "Completion", "Source", "From", "Row"); + + for (Method method : EsqlBaseParserVisitor.class.getMethods()) { + String methodName = method.getName(); + Matcher matcher = pattern.matcher(methodName); + if (matcher.matches()) { + String className = matcher.group(1); + if (ignoredClasses.contains(className)) { + continue; + } + String commandName = commandNameMapper.getOrDefault(className, className.toUpperCase(Locale.ROOT)); + if (commandClassNameMapper.containsKey(className)) { + className = commandClassNameMapper.get(className); + if (commandNameMapper.containsKey(className)) { + commandName = commandNameMapper.get(className); + } + } + try { + String fullClassName = commandPackageMapper.getOrDefault(className, planPackage) + "." + className; + Class candidateClass = Class.forName(fullClassName); + + if (LogicalPlan.class.isAssignableFrom(candidateClass)) { + commandClasses.put(commandName, candidateClass.asSubclass(LogicalPlan.class)); + } else { + log.info("Class " + className + " does NOT extend LogicalPlan."); + } + } catch (ClassNotFoundException e) { + log.info("Class " + className + " not found."); + } + } + } + return commandClasses; + } + + private static LogicalPlan createInstance(Class clazz, LogicalPlan child) throws InvocationTargetException, + InstantiationException, IllegalAccessException { + Source source = Source.EMPTY; + + // hard coded cases where the first two parameters are not Source and child LogicalPlan + switch (clazz.getSimpleName()) { + case "Grok" -> { + return new Grok(source, child, null, null, List.of()); + } + case "InlineStats" -> { + return new InlineStats(source, new Aggregate(Source.EMPTY, child, Aggregate.AggregateType.STANDARD, null, null)); + } + case "LookupJoin" -> { + return new LookupJoin(source, child, child, List.of()); + } + case "Limit" -> { + return new Limit(source, null, child); + } + } + + // For all others, find the constructor that takes Source and LogicalPlan as the first two parameters + Constructor[] constructors = clazz.getConstructors(); + + Constructor constructor = Arrays.stream(constructors).filter(c -> { + Class[] params = c.getParameterTypes(); + return params.length > 1 && Source.class.isAssignableFrom(params[0]) && LogicalPlan.class.isAssignableFrom(params[1]); + }) + .min(Comparator.comparingInt(c -> c.getParameterTypes().length)) + .orElseThrow(() -> new IllegalArgumentException("No suitable constructor found for class " + clazz.getName())); + + Class[] paramTypes = constructor.getParameterTypes(); + Object[] args = new Object[paramTypes.length]; + args[0] = source; + args[1] = child; + log.info("Creating instance of " + clazz.getName() + " with constructor: " + constructor); + return (LogicalPlan) constructor.newInstance(args); + } +}