From 2ad1f54a280a4a73205d255171d6171e0c8737ad Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 15 Oct 2025 09:01:17 +0200 Subject: [PATCH 1/3] Fix label filter for proper prefix regex --- .../rules/logical/promql/AutomatonUtils.java | 20 ++++-- .../TranslatePromqlToTimeSeriesAggregate.java | 4 ++ .../PromqlLogicalPlanOptimizerTests.java | 62 +++++++++++++++++-- .../logical/promql/AutomatonUtilsTests.java | 16 +++++ 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtils.java index 5f3b4f3115f92..1638e5f04fe67 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtils.java @@ -133,10 +133,12 @@ private static String removeEndingAnchor(String normalized) { */ public static class PatternFragment { public enum Type { - EXACT, // Exact literal match - PREFIX, // Starts with literal - SUFFIX, // Ends with literal - REGEX // Complex regex pattern + EXACT, // Exact literal match + PREFIX, // Starts with literal + PROPER_PREFIX, // Starts with literal but not the literal itself + SUFFIX, // Ends with literal + PROPER_SUFFIX, // Ends with literal but not the literal itself + REGEX // Complex regex pattern } private final Type type; @@ -183,7 +185,10 @@ private static PatternFragment classifyPart(String part) { // Suffix pattern: .*suffix String suffix = trimmed.substring(2); if (isLiteral(suffix)) { - return new PatternFragment(PatternFragment.Type.SUFFIX, suffix); + return new PatternFragment( + trimmed.startsWith(".*") ? PatternFragment.Type.SUFFIX : PatternFragment.Type.PROPER_SUFFIX, + suffix + ); } // Complex suffix pattern - fallback to REGEX return new PatternFragment(PatternFragment.Type.REGEX, part.trim()); @@ -193,7 +198,10 @@ private static PatternFragment classifyPart(String part) { // Prefix pattern: prefix.* String prefix = trimmed.substring(0, trimmed.length() - 2); if (isLiteral(prefix)) { - return new PatternFragment(PatternFragment.Type.PREFIX, prefix); + return new PatternFragment( + trimmed.endsWith(".*") ? PatternFragment.Type.PREFIX : PatternFragment.Type.PROPER_PREFIX, + prefix + ); } // Complex prefix pattern - fallback to REGEX return new PatternFragment(PatternFragment.Type.REGEX, part.trim()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/TranslatePromqlToTimeSeriesAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/TranslatePromqlToTimeSeriesAggregate.java index 5820805805bfd..feadb77f6552b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/TranslatePromqlToTimeSeriesAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/TranslatePromqlToTimeSeriesAggregate.java @@ -21,9 +21,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; import org.elasticsearch.xpack.esql.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.expression.promql.function.PromqlFunctionRegistry; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; import org.elasticsearch.xpack.esql.optimizer.rules.logical.TranslateTimeSeriesAggregate; @@ -344,7 +346,9 @@ private static Expression translatePatternFragment(Source source, Expression fie return switch (fragment.type()) { case EXACT -> new Equals(source, field, value); case PREFIX -> new StartsWith(source, field, value); + case PROPER_PREFIX -> new And(source, new NotEquals(source, field, value), new StartsWith(source, field, value)); case SUFFIX -> new EndsWith(source, field, value); + case PROPER_SUFFIX -> new And(source, new NotEquals(source, field, value), new EndsWith(source, field, value)); case REGEX -> new RLike(source, field, new RLikePattern(fragment.value())); }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java index 6a335161d6734..b72a3c0da4b0b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java @@ -12,10 +12,15 @@ import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.AbstractLogicalPlanOptimizerTests; +import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.junit.BeforeClass; import org.junit.Ignore; @@ -25,9 +30,10 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyInferenceResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; //@TestLogging(value="org.elasticsearch.xpack.esql:TRACE", reason="debug tests") -@Ignore("Proper assertions need to be added") public class PromqlLogicalPlanOptimizerTests extends AbstractLogicalPlanOptimizerTests { private static final String PARAM_FORMATTING = "%1$s"; @@ -52,6 +58,7 @@ public static void initTest() { ); } + @Ignore("Proper assertions need to be added") public void testExplainPromql() { // TS metrics-hostmetricsreceiver.otel-default // | WHERE @timestamp >= \"{{from | minus .benchmark.duration}}\" AND @timestamp <=\"{{from}}\" @@ -67,6 +74,7 @@ public void testExplainPromql() { System.out.println(plan); } + @Ignore("Proper assertions need to be added") public void testExplainPromqlSimple() { // TS metrics-hostmetricsreceiver.otel-default // | WHERE @timestamp >= \"{{from | minus .benchmark.duration}}\" AND @timestamp <=\"{{from}}\" @@ -81,6 +89,8 @@ public void testExplainPromqlSimple() { System.out.println(plan); } + + @Ignore("Proper assertions need to be added") public void testAvgAvgOverTimeOutput() { // TS metrics-hostmetricsreceiver.otel-default // | WHERE @timestamp >= \"{{from | minus .benchmark.duration}}\" AND @timestamp <=\"{{from}}\" @@ -94,6 +104,7 @@ public void testAvgAvgOverTimeOutput() { System.out.println(plan); } + @Ignore("Proper assertions need to be added") public void testTSAvgAvgOverTimeOutput() { // TS metrics-hostmetricsreceiver.otel-default // | STATS AVG(AVG_OVER_TIME(`metrics.system.memory.utilization`)) BY host.name, TBUCKET(1h) | LIMIT 10000" @@ -106,6 +117,7 @@ public void testTSAvgAvgOverTimeOutput() { System.out.println(plan); } + @Ignore("Proper assertions need to be added") public void testTSAvgWithoutByDimension() { // TS metrics-hostmetricsreceiver.otel-default // | STATS AVG(AVG_OVER_TIME(`metrics.system.memory.utilization`)) BY TBUCKET(1h) | LIMIT 10000" @@ -118,6 +130,7 @@ public void testTSAvgWithoutByDimension() { System.out.println(plan); } + @Ignore("Proper assertions need to be added") public void testPromqlAvgWithoutByDimension() { // TS metrics-hostmetricsreceiver.otel-default // | STATS AVG(AVG_OVER_TIME(`metrics.system.memory.utilization`)) BY TBUCKET(1h) | LIMIT 10000" @@ -130,6 +143,7 @@ public void testPromqlAvgWithoutByDimension() { System.out.println(plan); } + @Ignore("Proper assertions need to be added") public void testRangeSelector() { // TS metrics-hostmetricsreceiver.otel-default // | WHERE @timestamp >= \"{{from | minus .benchmark.duration}}\" AND @timestamp <=\"{{from}}\" @@ -142,6 +156,7 @@ public void testRangeSelector() { System.out.println(plan); } + @Ignore("Proper assertions need to be added") public void testRate() { // TS metrics-hostmetricsreceiver.otel-default // | WHERE @timestamp >= \"{{from | minus .benchmark.duration}}\" AND @timestamp <= \"{{from}}\" @@ -164,11 +179,14 @@ public void testLabelSelector() { String testQuery = """ TS k8s | promql - max by (pod)(avg_over_time(network.total_bytes_in{pod=~"host-0|host-1|host-2"}[5m])) - + max by (pod) (avg_over_time(network.bytes_in{pod=~"host-0|host-1|host-2"}[5m])) """; var plan = planPromql(testQuery); + var filters = plan.collect(Filter.class::isInstance); + assertThat(filters, hasSize(1)); + var filter = (Filter) filters.getFirst(); + assertThat(filter.condition().anyMatch(In.class::isInstance), equalTo(true)); System.out.println(plan); } @@ -179,14 +197,46 @@ public void testLabelSelectorPrefix() { String testQuery = """ TS k8s | promql - avg by (pod)(avg_over_time(network.total_bytes_in{pod=~"host-.*"}[5m])) - + avg by (pod) (avg_over_time(network.bytes_in{pod=~"host-.*"}[5m])) """; var plan = planPromql(testQuery); + var filters = plan.collect(Filter.class::isInstance); + assertThat(filters, hasSize(1)); + var filter = (Filter) filters.getFirst(); + assertThat(filter.condition().anyMatch(StartsWith.class::isInstance), equalTo(true)); + assertThat(filter.condition().anyMatch(NotEquals.class::isInstance), equalTo(false)); System.out.println(plan); } + public void testLabelSelectorProperPrefix() { + var plan = planPromql(""" + TS k8s + | promql avg(avg_over_time(network.bytes_in{pod=~"host-.+"}[1h])) + | LIMIT 1000 + """); + + var filters = plan.collect(Filter.class::isInstance); + assertThat(filters, hasSize(1)); + var filter = (Filter) filters.getFirst(); + assertThat(filter.condition().anyMatch(StartsWith.class::isInstance), equalTo(true)); + assertThat(filter.condition().anyMatch(NotEquals.class::isInstance), equalTo(true)); + } + + public void testLabelSelectorRegex() { + var plan = planPromql(""" + TS k8s + | promql avg(avg_over_time(network.bytes_in{pod=~"[a-z]+"}[1h])) + | LIMIT 1000 + """); + + var filters = plan.collect(Filter.class::isInstance); + assertThat(filters, hasSize(1)); + var filter = (Filter) filters.getFirst(); + assertThat(filter.condition().anyMatch(RegexMatch.class::isInstance), equalTo(true)); + } + + @Ignore("Proper assertions need to be added") public void testFsUsageTop5() { // TS metrics-hostmetricsreceiver.otel-default | WHERE @timestamp >= \"{{from | minus .benchmark.duration}}\" AND @timestamp <= \"{{from}}\" // | WHERE attributes.state IN (\"used\", \"free\") @@ -208,7 +258,7 @@ sum by (host.name, mountpoint) (last_over_time(system.filesystem.usage{state=~"u protected LogicalPlan planPromql(String query) { - var analyzed = tsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); + var analyzed = tsAnalyzer.analyze(parser.createStatement(query)); System.out.println(analyzed); var optimized = logicalOptimizer.optimize(analyzed); System.out.println(optimized); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java index 50f819463d2c1..403e3d528d274 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java @@ -14,6 +14,8 @@ import static org.elasticsearch.xpack.esql.optimizer.rules.logical.promql.AutomatonUtils.PatternFragment.Type.EXACT; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.promql.AutomatonUtils.PatternFragment.Type.PREFIX; +import static org.elasticsearch.xpack.esql.optimizer.rules.logical.promql.AutomatonUtils.PatternFragment.Type.PROPER_PREFIX; +import static org.elasticsearch.xpack.esql.optimizer.rules.logical.promql.AutomatonUtils.PatternFragment.Type.PROPER_SUFFIX; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.promql.AutomatonUtils.PatternFragment.Type.REGEX; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.promql.AutomatonUtils.PatternFragment.Type.SUFFIX; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.promql.AutomatonUtils.extractFragments; @@ -67,6 +69,20 @@ public void testExtractFragments_MixedAlternation() { assertFragments(fragments, expected); } + public void testExtractFragments_ProperPrefixSuffixAlternation() { + List fragments = extractFragments("prod-.+|.+-dev"); + + assertThat(fragments, notNullValue()); + assertThat(fragments, hasSize(2)); + + Object[][] expected = { + { PROPER_PREFIX, "prod-" }, + { PROPER_SUFFIX, "-dev" } + }; + + assertFragments(fragments, expected); + } + public void testExtractFragments_HomogeneousExactAlternation() { // All exact values List fragments = extractFragments("api|web|service"); From 5609698bc28263efebe6383c4cdedd2ffb64fc08 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 15 Oct 2025 09:21:55 +0200 Subject: [PATCH 2/3] Apply spotless suggestions --- .../optimizer/rules/logical/promql/AutomatonUtilsTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java index 1423b808db684..8cee71294713f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/AutomatonUtilsTests.java @@ -71,10 +71,7 @@ public void testExtractFragments_ProperPrefixSuffixAlternation() { assertThat(fragments, notNullValue()); assertThat(fragments, hasSize(2)); - Object[][] expected = { - { PROPER_PREFIX, "prod-" }, - { PROPER_SUFFIX, "-dev" } - }; + Object[][] expected = { { PROPER_PREFIX, "prod-" }, { PROPER_SUFFIX, "-dev" } }; assertFragments(fragments, expected); } From 330418f6f0def5a0e704279de6a782730bcb59f9 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 29 Oct 2025 10:16:14 +0100 Subject: [PATCH 3/3] Fix query formatting --- .../esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java index 3c66bacf3c164..38b75599a3f3f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java @@ -178,7 +178,8 @@ public void testLabelSelector() { String testQuery = """ TS k8s | promql time now ( - max by (pod) (avg_over_time(network.bytes_in{pod=~"host-0|host-1|host-2"}[5m]))) + max by (pod) (avg_over_time(network.bytes_in{pod=~"host-0|host-1|host-2"}[5m])) + ) """; var plan = planPromql(testQuery);