diff --git a/docs/changelog/134309.yaml b/docs/changelog/134309.yaml new file mode 100644 index 0000000000000..bad463f3ba14d --- /dev/null +++ b/docs/changelog/134309.yaml @@ -0,0 +1,5 @@ +pr: 134309 +summary: Telemetry with inlinestats +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index df83feeac9f13..957b6e1b52447 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -100,6 +100,7 @@ import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Fork; +import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.Limit; @@ -171,6 +172,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; +import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.STATS; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; /** @@ -1401,6 +1403,24 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { if (plan.collectFirstChildren(Limit.class::isInstance).isEmpty() == false) { b.set(LIMIT.ordinal()); } + + // count only the Aggregate (STATS command) that is "standalone" not also the one that is part of an INLINESTATS command + if (plan instanceof Aggregate) { + b.set(STATS.ordinal()); + } else { + plan.forEachDownMayReturnEarly((p, breakEarly) -> { + if (p instanceof InlineStats) { + return; + } + for (var c : p.children()) { + if (c instanceof Aggregate) { + b.set(STATS.ordinal()); + breakEarly.set(true); + return; + } + } + }); + } plan.forEachDown(p -> FeatureMetric.set(p, b)); return b; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java index f7ebaed247e6e..9645d4af28c4a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java @@ -51,7 +51,8 @@ public enum FeatureMetric { GROK(Grok.class::isInstance), LIMIT(plan -> false), // the limit is checked in Analyzer.gatherPreAnalysisMetrics, because it has a more complex and general check SORT(OrderBy.class::isInstance), - STATS(Aggregate.class::isInstance), + // the STATS is checked in Analyzer.gatherPreAnalysisMetrics, because it can also be part of an inlinestats command + STATS(plan -> false), WHERE(Filter.class::isInstance), ENRICH(Enrich.class::isInstance), EXPLAIN(Explain.class::isInstance), @@ -81,7 +82,8 @@ public enum FeatureMetric { EsqlProject.class, Project.class, Limit.class, // LIMIT is managed in another way, see above - FuseScoreEval.class + FuseScoreEval.class, + Aggregate.class // STATS is managed in another way, see above ); private Predicate planCheck; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java index 1f3047262c94d..fde5c16d0a71d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.common.stats.Counters; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; @@ -29,8 +30,10 @@ import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.EVAL; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.FROM; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.GROK; +import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.INLINESTATS; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.KEEP; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; +import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.MV_EXPAND; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.RENAME; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.ROW; @@ -62,7 +65,8 @@ public void testDissectQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("concat", c)); } @@ -83,7 +87,8 @@ public void testEvalQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("length", c)); } @@ -104,7 +109,8 @@ public void testGrokQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("concat", c)); } @@ -125,6 +131,8 @@ public void testLimitQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testSortQuery() { @@ -144,6 +152,8 @@ public void testSortQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testStatsQuery() { @@ -163,7 +173,8 @@ public void testStatsQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("max", c)); } @@ -184,6 +195,8 @@ public void testWhereQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testTwoWhereQuery() { @@ -203,6 +216,8 @@ public void testTwoWhereQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testTwoQueriesExecuted() { @@ -242,6 +257,8 @@ public void testTwoQueriesExecuted() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("length", c)); assertEquals(1, function("concat", c)); @@ -325,7 +342,8 @@ public void testEnrich() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("to_string", c)); } @@ -355,6 +373,8 @@ public void testMvExpand() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); } public void testShowInfo() { @@ -374,7 +394,8 @@ public void testShowInfo() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("count", c)); } @@ -395,6 +416,8 @@ public void testRow() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); } public void testDropAndRename() { @@ -414,7 +437,8 @@ public void testDropAndRename() { assertEquals(1L, drop(c)); assertEquals(0, keep(c)); assertEquals(1L, rename(c)); - + assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("count", c)); } @@ -440,6 +464,8 @@ public void testKeep() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); } public void testCategorize() { @@ -463,10 +489,118 @@ public void testCategorize() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("count", c)); assertEquals(1, function("categorize", c)); } + public void testInlinestatsStandalone() { + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); + Counters c = esql(""" + from employees + | inlinestats max(salary) by gender + | where languages is not null"""); + assertEquals(0, dissect(c)); + assertEquals(0, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(0, stats(c)); + assertEquals(1L, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(1L, inlinestats(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(1, function("max", c)); + } + + public void testInlinestatsWithOtherStats() { + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); + Counters c = esql(""" + from employees + | inlinestats m = max(salary) by gender + | where languages is not null + | stats max(m) by languages"""); + assertEquals(0, dissect(c)); + assertEquals(0, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(1L, stats(c)); + assertEquals(1L, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(1L, inlinestats(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(1, function("max", c)); + } + + public void testBinaryPlanAfterStats() { + Counters c = esql(""" + from employees + | eval language_code = languages + | stats m = max(salary) by language_code + | lookup join languages_lookup on language_code"""); + assertEquals(0, dissect(c)); + assertEquals(1L, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(1L, stats(c)); + assertEquals(0, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); + assertEquals(1L, lookupjoin(c)); + assertEquals(1, function("max", c)); + } + + public void testBinaryPlanAfterInlinestats() { + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); + Counters c = esql(""" + from employees + | eval language_code = languages + | inlinestats m = max(salary) by language_code + | lookup join languages_lookup on language_code"""); + assertEquals(0, dissect(c)); + assertEquals(1L, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(0, stats(c)); + assertEquals(0, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(1L, inlinestats(c)); + assertEquals(1L, lookupjoin(c)); + assertEquals(1, function("max", c)); + } + private long dissect(Counters c) { return c.get(FEATURES_PREFIX + DISSECT); } @@ -527,6 +661,14 @@ private long rename(Counters c) { return c.get(FEATURES_PREFIX + RENAME); } + private long inlinestats(Counters c) { + return c.get(FEATURES_PREFIX + INLINESTATS); + } + + private long lookupjoin(Counters c) { + return c.get(FEATURES_PREFIX + LOOKUP_JOIN); + } + private long function(String function, Counters c) { return c.get(FUNC_PREFIX + function); }