Skip to content

ESQL: telemetry with inlinestats #134309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Sep 10, 2025
Merged

ESQL: telemetry with inlinestats #134309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7fcf2b4
telemetry with inlinestats
astefan Sep 8, 2025
c92de87
Update docs/changelog/134309.yaml
astefan Sep 8, 2025
fb2cec4
Merge branch 'main' into inlinestats_telemetry
astefan Sep 8, 2025
587682f
Merge branch 'main' into inlinestats_telemetry
astefan Sep 8, 2025
807fffd
Address reviews
astefan Sep 9, 2025
80fe9c8
Merge branch 'main' of https://github.com/elastic/elasticsearch into …
astefan Sep 9, 2025
22ef23a
Merge branch 'inlinestats_telemetry' of https://github.com/astefan/el…
astefan Sep 9, 2025
4ecd9cd
Merge branch 'main' into inlinestats_telemetry
astefan Sep 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/134309.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 134309
summary: Telemetry with inlinestats
area: ES|QL
type: bug
issues: []
20 changes: 20 additions & 0 deletions x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
import org.elasticsearch.xpack.esql.plan.logical.Eval;
import org.elasticsearch.xpack.esql.plan.logical.Fork;
import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
import org.elasticsearch.xpack.esql.plan.logical.Insist;
import org.elasticsearch.xpack.esql.plan.logical.Keep;
import org.elasticsearch.xpack.esql.plan.logical.Limit;
Expand Down Expand Up @@ -171,6 +172,7 @@
import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.STATS;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount;

/**
Expand Down Expand Up @@ -1401,6 +1403,24 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) {
if (plan.collectFirstChildren(Limit.class::isInstance).isEmpty() == false) {
b.set(LIMIT.ordinal());
}

// count only the Aggregate (STATS command) that is "standalone" not also the one that is part of an INLINESTATS command
if (plan instanceof Aggregate) {
b.set(STATS.ordinal());
} else {
plan.forEachDownMayReturnEarly((p, breakEarly) -> {
if (p instanceof InlineStats) {
return;
}
for (var c : p.children()) {
if (c instanceof Aggregate) {
b.set(STATS.ordinal());
breakEarly.set(true);
return;
}
}
});
}
plan.forEachDown(p -> FeatureMetric.set(p, b));
return b;
}
Expand Down
6 changes: 4 additions & 2 deletions x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ public enum FeatureMetric {
GROK(Grok.class::isInstance),
LIMIT(plan -> false), // the limit is checked in Analyzer.gatherPreAnalysisMetrics, because it has a more complex and general check
SORT(OrderBy.class::isInstance),
STATS(Aggregate.class::isInstance),
// the STATS is checked in Analyzer.gatherPreAnalysisMetrics, because it can also be part of an inlinestats command
STATS(plan -> false),
WHERE(Filter.class::isInstance),
ENRICH(Enrich.class::isInstance),
EXPLAIN(Explain.class::isInstance),
Expand Down Expand Up @@ -81,7 +82,8 @@ public enum FeatureMetric {
EsqlProject.class,
Project.class,
Limit.class, // LIMIT is managed in another way, see above
FuseScoreEval.class
FuseScoreEval.class,
Aggregate.class // STATS is managed in another way, see above
);

private Predicate<LogicalPlan> planCheck;
Expand Down
156 changes: 149 additions & 7 deletions ...lugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.elasticsearch.xpack.esql.analysis.Verifier;
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
Expand All @@ -29,8 +30,10 @@
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.EVAL;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.FROM;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.GROK;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.INLINESTATS;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.KEEP;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.MV_EXPAND;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.RENAME;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.ROW;
Expand Down Expand Up @@ -62,7 +65,8 @@ public void testDissectQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));

assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
assertEquals(1, function("concat", c));
}

Expand All @@ -83,7 +87,8 @@ public void testEvalQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));

assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
assertEquals(1, function("length", c));
}

Expand All @@ -104,7 +109,8 @@ public void testGrokQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));

assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
assertEquals(1, function("concat", c));
}

Expand All @@ -125,6 +131,8 @@ public void testLimitQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
}

public void testSortQuery() {
Expand All @@ -144,6 +152,8 @@ public void testSortQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
}

public void testStatsQuery() {
Expand All @@ -163,7 +173,8 @@ public void testStatsQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));

assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
assertEquals(1, function("max", c));
}

Expand All @@ -184,6 +195,8 @@ public void testWhereQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
}

public void testTwoWhereQuery() {
Expand All @@ -203,6 +216,8 @@ public void testTwoWhereQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));
}

public void testTwoQueriesExecuted() {
Expand Down Expand Up @@ -242,6 +257,8 @@ public void testTwoQueriesExecuted() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, inlinestats(c));

assertEquals(1, function("length", c));
assertEquals(1, function("concat", c));
Expand Down Expand Up @@ -325,7 +342,8 @@ public void testEnrich() {
assertEquals(0, drop(c));
assertEquals(1L, keep(c));
assertEquals(0, rename(c));

assertEquals(0, inlinestats(c));
assertEquals(0, lookupjoin(c));
assertEquals(1, function("to_string", c));
}

Expand Down Expand Up @@ -355,6 +373,8 @@ public void testMvExpand() {
assertEquals(0, drop(c));
assertEquals(1L, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlinestats(c));
assertEquals(0, lookupjoin(c));
}

public void testShowInfo() {
Expand All @@ -374,7 +394,8 @@ public void testShowInfo() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));

assertEquals(0, inlinestats(c));
assertEquals(0, lookupjoin(c));
assertEquals(1, function("count", c));
}

Expand All @@ -395,6 +416,8 @@ public void testRow() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlinestats(c));
assertEquals(0, lookupjoin(c));
}

public void testDropAndRename() {
Expand All @@ -414,7 +437,8 @@ public void testDropAndRename() {
assertEquals(1L, drop(c));
assertEquals(0, keep(c));
assertEquals(1L, rename(c));

assertEquals(0, inlinestats(c));
assertEquals(0, lookupjoin(c));
assertEquals(1, function("count", c));
}

Expand All @@ -440,6 +464,8 @@ public void testKeep() {
assertEquals(0, drop(c));
assertEquals(1L, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlinestats(c));
assertEquals(0, lookupjoin(c));
}

public void testCategorize() {
Expand All @@ -463,10 +489,118 @@ public void testCategorize() {
assertEquals(0, drop(c));
assertEquals(1L, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlinestats(c));
assertEquals(0, lookupjoin(c));
assertEquals(1, function("count", c));
assertEquals(1, function("categorize", c));
}

public void testInlinestatsStandalone() {
assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled());
Counters c = esql("""
from employees
| inlinestats max(salary) by gender
| where languages is not null""");
assertEquals(0, dissect(c));
assertEquals(0, eval(c));
assertEquals(0, grok(c));
assertEquals(0, limit(c));
assertEquals(0, sort(c));
assertEquals(0, stats(c));
assertEquals(1L, where(c));
assertEquals(0, enrich(c));
assertEquals(0, mvExpand(c));
assertEquals(0, show(c));
assertEquals(0, row(c));
assertEquals(1L, from(c));
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(1L, inlinestats(c));
assertEquals(0, lookupjoin(c));
assertEquals(1, function("max", c));
Comment on lines +504 to +521
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be worth at some point to refactor this list and somehow only provide the non-zero counters to a checking function that would also assert on all the rest being 0. Maybe when we'd consider tapping the parser for usage.

}

public void testInlinestatsWithOtherStats() {
assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled());
Counters c = esql("""
from employees
| inlinestats m = max(salary) by gender
| where languages is not null
| stats max(m) by languages""");
assertEquals(0, dissect(c));
assertEquals(0, eval(c));
assertEquals(0, grok(c));
assertEquals(0, limit(c));
assertEquals(0, sort(c));
assertEquals(1L, stats(c));
assertEquals(1L, where(c));
assertEquals(0, enrich(c));
assertEquals(0, mvExpand(c));
assertEquals(0, show(c));
assertEquals(0, row(c));
assertEquals(1L, from(c));
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(1L, inlinestats(c));
assertEquals(0, lookupjoin(c));
assertEquals(1, function("max", c));
}

public void testBinaryPlanAfterStats() {
Counters c = esql("""
from employees
| eval language_code = languages
| stats m = max(salary) by language_code
| lookup join languages_lookup on language_code""");
assertEquals(0, dissect(c));
assertEquals(1L, eval(c));
assertEquals(0, grok(c));
assertEquals(0, limit(c));
assertEquals(0, sort(c));
assertEquals(1L, stats(c));
assertEquals(0, where(c));
assertEquals(0, enrich(c));
assertEquals(0, mvExpand(c));
assertEquals(0, show(c));
assertEquals(0, row(c));
assertEquals(1L, from(c));
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlinestats(c));
assertEquals(1L, lookupjoin(c));
assertEquals(1, function("max", c));
}

public void testBinaryPlanAfterInlinestats() {
assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled());
Counters c = esql("""
from employees
| eval language_code = languages
| inlinestats m = max(salary) by language_code
| lookup join languages_lookup on language_code""");
assertEquals(0, dissect(c));
assertEquals(1L, eval(c));
assertEquals(0, grok(c));
assertEquals(0, limit(c));
assertEquals(0, sort(c));
assertEquals(0, stats(c));
assertEquals(0, where(c));
assertEquals(0, enrich(c));
assertEquals(0, mvExpand(c));
assertEquals(0, show(c));
assertEquals(0, row(c));
assertEquals(1L, from(c));
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(1L, inlinestats(c));
assertEquals(1L, lookupjoin(c));
assertEquals(1, function("max", c));
}

private long dissect(Counters c) {
return c.get(FEATURES_PREFIX + DISSECT);
}
Expand Down Expand Up @@ -527,6 +661,14 @@ private long rename(Counters c) {
return c.get(FEATURES_PREFIX + RENAME);
}

private long inlinestats(Counters c) {
return c.get(FEATURES_PREFIX + INLINESTATS);
}

private long lookupjoin(Counters c) {
return c.get(FEATURES_PREFIX + LOOKUP_JOIN);
}

private long function(String function, Counters c) {
return c.get(FUNC_PREFIX + function);
}
Expand Down