From 7fcf2b479dc26c9546915ddf6f2d685977d33f02 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 8 Sep 2025 19:04:16 +0300 Subject: [PATCH 1/3] telemetry with inlinestats --- .../xpack/esql/analysis/Analyzer.java | 15 +++++ .../xpack/esql/telemetry/FeatureMetric.java | 6 +- .../esql/telemetry/VerifierMetricsTests.java | 67 ++++++++++++++++++- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index df83feeac9f13..e7f81221f4b5c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -100,6 +100,7 @@ import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Fork; +import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.Limit; @@ -108,6 +109,7 @@ import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; +import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.fuse.Fuse; import org.elasticsearch.xpack.esql.plan.logical.fuse.FuseScoreEval; @@ -171,6 +173,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; +import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.STATS; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; /** @@ -1401,6 +1404,18 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { if (plan.collectFirstChildren(Limit.class::isInstance).isEmpty() == false) { b.set(LIMIT.ordinal()); } + + // count only the Aggregate (STATS command) that is "standalone" not also the one that is part of an INLINESTATS command + if (plan instanceof Aggregate) { + b.set(STATS.ordinal()); + } else { + plan.forEachDownMayReturnEarly((p, breakEarly) -> { + if (p instanceof UnaryPlan up && up.child() instanceof Aggregate && p instanceof InlineStats == false) { + b.set(STATS.ordinal()); + breakEarly.set(true); + } + }); + } plan.forEachDown(p -> FeatureMetric.set(p, b)); return b; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java index f7ebaed247e6e..9645d4af28c4a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java @@ -51,7 +51,8 @@ public enum FeatureMetric { GROK(Grok.class::isInstance), LIMIT(plan -> false), // the limit is checked in Analyzer.gatherPreAnalysisMetrics, because it has a more complex and general check SORT(OrderBy.class::isInstance), - STATS(Aggregate.class::isInstance), + // the STATS is checked in Analyzer.gatherPreAnalysisMetrics, because it can also be part of an inlinestats command + STATS(plan -> false), WHERE(Filter.class::isInstance), ENRICH(Enrich.class::isInstance), EXPLAIN(Explain.class::isInstance), @@ -81,7 +82,8 @@ public enum FeatureMetric { EsqlProject.class, Project.class, Limit.class, // LIMIT is managed in another way, see above - FuseScoreEval.class + FuseScoreEval.class, + Aggregate.class // STATS is managed in another way, see above ); private Predicate planCheck; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java index 1f3047262c94d..e74e4f6612b6d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.common.stats.Counters; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; @@ -29,6 +30,7 @@ import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.EVAL; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.FROM; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.GROK; +import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.INLINESTATS; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.KEEP; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.MV_EXPAND; @@ -325,7 +327,7 @@ public void testEnrich() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, inlinestats(c)); assertEquals(1, function("to_string", c)); } @@ -355,6 +357,7 @@ public void testMvExpand() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); } public void testShowInfo() { @@ -374,7 +377,7 @@ public void testShowInfo() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, inlinestats(c)); assertEquals(1, function("count", c)); } @@ -395,6 +398,7 @@ public void testRow() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); } public void testDropAndRename() { @@ -414,7 +418,7 @@ public void testDropAndRename() { assertEquals(1L, drop(c)); assertEquals(0, keep(c)); assertEquals(1L, rename(c)); - + assertEquals(0, inlinestats(c)); assertEquals(1, function("count", c)); } @@ -440,6 +444,7 @@ public void testKeep() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); } public void testCategorize() { @@ -463,10 +468,62 @@ public void testCategorize() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("count", c)); assertEquals(1, function("categorize", c)); } + public void testInlinestatsStandalone() { + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); + Counters c = esql(""" + from employees + | inlinestats max(salary) by gender + | where languages is not null"""); + assertEquals(0, dissect(c)); + assertEquals(0, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(0, stats(c)); + assertEquals(1L, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(1L, inlinestats(c)); + assertEquals(1, function("max", c)); + } + + public void testInlinestatsWithOtherStats() { + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); + Counters c = esql(""" + from employees + | inlinestats m = max(salary) by gender + | where languages is not null + | stats max(m) by languages"""); + assertEquals(0, dissect(c)); + assertEquals(0, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(1L, stats(c)); + assertEquals(1L, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(1L, inlinestats(c)); + assertEquals(1, function("max", c)); + } + private long dissect(Counters c) { return c.get(FEATURES_PREFIX + DISSECT); } @@ -527,6 +584,10 @@ private long rename(Counters c) { return c.get(FEATURES_PREFIX + RENAME); } + private long inlinestats(Counters c) { + return c.get(FEATURES_PREFIX + INLINESTATS); + } + private long function(String function, Counters c) { return c.get(FUNC_PREFIX + function); } From c92de87a41a09c0d7e922c1576254982dd518b46 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 8 Sep 2025 19:06:19 +0300 Subject: [PATCH 2/3] Update docs/changelog/134309.yaml --- docs/changelog/134309.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/134309.yaml diff --git a/docs/changelog/134309.yaml b/docs/changelog/134309.yaml new file mode 100644 index 0000000000000..bad463f3ba14d --- /dev/null +++ b/docs/changelog/134309.yaml @@ -0,0 +1,5 @@ +pr: 134309 +summary: Telemetry with inlinestats +area: ES|QL +type: bug +issues: [] From 807fffd6a8e19dcc8a476d1c4a9797b718c91df5 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 9 Sep 2025 16:28:15 +0300 Subject: [PATCH 3/3] Address reviews --- .../xpack/esql/analysis/Analyzer.java | 13 ++- .../esql/telemetry/VerifierMetricsTests.java | 89 ++++++++++++++++++- 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index e7f81221f4b5c..957b6e1b52447 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -109,7 +109,6 @@ import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; -import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.fuse.Fuse; import org.elasticsearch.xpack.esql.plan.logical.fuse.FuseScoreEval; @@ -1410,9 +1409,15 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { b.set(STATS.ordinal()); } else { plan.forEachDownMayReturnEarly((p, breakEarly) -> { - if (p instanceof UnaryPlan up && up.child() instanceof Aggregate && p instanceof InlineStats == false) { - b.set(STATS.ordinal()); - breakEarly.set(true); + if (p instanceof InlineStats) { + return; + } + for (var c : p.children()) { + if (c instanceof Aggregate) { + b.set(STATS.ordinal()); + breakEarly.set(true); + return; + } } }); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java index e74e4f6612b6d..fde5c16d0a71d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java @@ -33,6 +33,7 @@ import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.INLINESTATS; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.KEEP; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; +import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.MV_EXPAND; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.RENAME; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.ROW; @@ -64,7 +65,8 @@ public void testDissectQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("concat", c)); } @@ -85,7 +87,8 @@ public void testEvalQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("length", c)); } @@ -106,7 +109,8 @@ public void testGrokQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("concat", c)); } @@ -127,6 +131,8 @@ public void testLimitQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testSortQuery() { @@ -146,6 +152,8 @@ public void testSortQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testStatsQuery() { @@ -165,7 +173,8 @@ public void testStatsQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("max", c)); } @@ -186,6 +195,8 @@ public void testWhereQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testTwoWhereQuery() { @@ -205,6 +216,8 @@ public void testTwoWhereQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); } public void testTwoQueriesExecuted() { @@ -244,6 +257,8 @@ public void testTwoQueriesExecuted() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(0, inlinestats(c)); assertEquals(1, function("length", c)); assertEquals(1, function("concat", c)); @@ -328,6 +343,7 @@ public void testEnrich() { assertEquals(1L, keep(c)); assertEquals(0, rename(c)); assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("to_string", c)); } @@ -358,6 +374,7 @@ public void testMvExpand() { assertEquals(1L, keep(c)); assertEquals(0, rename(c)); assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); } public void testShowInfo() { @@ -378,6 +395,7 @@ public void testShowInfo() { assertEquals(0, keep(c)); assertEquals(0, rename(c)); assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("count", c)); } @@ -399,6 +417,7 @@ public void testRow() { assertEquals(0, keep(c)); assertEquals(0, rename(c)); assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); } public void testDropAndRename() { @@ -419,6 +438,7 @@ public void testDropAndRename() { assertEquals(0, keep(c)); assertEquals(1L, rename(c)); assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("count", c)); } @@ -445,6 +465,7 @@ public void testKeep() { assertEquals(1L, keep(c)); assertEquals(0, rename(c)); assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); } public void testCategorize() { @@ -469,6 +490,7 @@ public void testCategorize() { assertEquals(1L, keep(c)); assertEquals(0, rename(c)); assertEquals(0, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("count", c)); assertEquals(1, function("categorize", c)); } @@ -495,6 +517,7 @@ public void testInlinestatsStandalone() { assertEquals(0, keep(c)); assertEquals(0, rename(c)); assertEquals(1L, inlinestats(c)); + assertEquals(0, lookupjoin(c)); assertEquals(1, function("max", c)); } @@ -521,6 +544,60 @@ public void testInlinestatsWithOtherStats() { assertEquals(0, keep(c)); assertEquals(0, rename(c)); assertEquals(1L, inlinestats(c)); + assertEquals(0, lookupjoin(c)); + assertEquals(1, function("max", c)); + } + + public void testBinaryPlanAfterStats() { + Counters c = esql(""" + from employees + | eval language_code = languages + | stats m = max(salary) by language_code + | lookup join languages_lookup on language_code"""); + assertEquals(0, dissect(c)); + assertEquals(1L, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(1L, stats(c)); + assertEquals(0, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(0, inlinestats(c)); + assertEquals(1L, lookupjoin(c)); + assertEquals(1, function("max", c)); + } + + public void testBinaryPlanAfterInlinestats() { + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); + Counters c = esql(""" + from employees + | eval language_code = languages + | inlinestats m = max(salary) by language_code + | lookup join languages_lookup on language_code"""); + assertEquals(0, dissect(c)); + assertEquals(1L, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(0, stats(c)); + assertEquals(0, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(1L, from(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(1L, inlinestats(c)); + assertEquals(1L, lookupjoin(c)); assertEquals(1, function("max", c)); } @@ -588,6 +665,10 @@ private long inlinestats(Counters c) { return c.get(FEATURES_PREFIX + INLINESTATS); } + private long lookupjoin(Counters c) { + return c.get(FEATURES_PREFIX + LOOKUP_JOIN); + } + private long function(String function, Counters c) { return c.get(FUNC_PREFIX + function); }