From a43470b06cf6ac4f96f29d420ff6bc6a58620d30 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 20 Nov 2025 16:33:35 +0100 Subject: [PATCH 1/7] Additional data structures --- .../_nightly/esql/QueryPlanningBenchmark.java | 2 +- .../xpack/esql/EsqlTestUtils.java | 2 +- .../xpack/esql/action/LookupFromIndexIT.java | 2 +- .../xpack/esql/analysis/Analyzer.java | 2 ++ .../esql/enrich/EnrichPolicyResolver.java | 7 +++-- .../fulltext/QueryBuilderResolver.java | 2 +- .../xpack/esql/index/EsIndex.java | 5 +++- .../xpack/esql/index/IndexResolution.java | 4 +-- .../logical/SkipQueryOnEmptyMappings.java | 2 +- .../xpack/esql/plan/logical/EsRelation.java | 29 +++++++++++++++---- .../xpack/esql/planner/PlannerUtils.java | 2 +- .../xpack/esql/session/EsqlSession.java | 11 +++++-- .../xpack/esql/session/IndexResolver.java | 9 +++++- .../elasticsearch/xpack/esql/CsvTests.java | 14 +++++++-- .../esql/analysis/AnalyzerTestUtils.java | 6 +++- .../xpack/esql/analysis/AnalyzerTests.java | 12 ++++++-- .../enrich/LookupFromIndexOperatorTests.java | 2 +- .../xpack/esql/index/EsIndexGenerator.java | 21 ++++++++++---- .../AbstractLogicalPlanOptimizerTests.java | 11 ++++++- .../optimizer/LogicalPlanOptimizerTests.java | 4 +-- .../optimizer/PhysicalPlanOptimizerTests.java | 2 ++ .../PromqlLogicalPlanOptimizerTests.java | 4 ++- .../PushDownAndCombineFiltersTests.java | 2 +- .../logical/EsRelationSerializationTests.java | 13 +++++++-- .../ExchangeSinkExecSerializationTests.java | 10 ++++++- .../xpack/esql/session/EsqlCCSUtilsTests.java | 10 +++---- .../esql/tree/EsqlNodeSubclassTests.java | 3 ++ 27 files changed, 147 insertions(+), 46 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java index 51ccbd51eb940..dd12ce85b3bd8 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java @@ -102,7 +102,7 @@ public void setup() { mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE)); } - var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Set.of()); + var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Set.of()); var functionRegistry = new EsqlFunctionRegistry(); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 2f3fb694dba60..e8083fefbdca9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -324,7 +324,7 @@ public static EsRelation relation() { } public static EsRelation relation(IndexMode mode) { - return new EsRelation(EMPTY, randomIdentifier(), mode, Map.of(), List.of()); + return new EsRelation(EMPTY, randomIdentifier(), mode, Map.of(), Map.of(), Map.of(), List.of()); } /** diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java index a5c1b12e45044..2544edda6f53d 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java @@ -246,7 +246,7 @@ private PhysicalPlan buildGreaterThanFilter(long value) { new EsField("l", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) ); Expression greaterThan = new GreaterThan(Source.EMPTY, filterAttribute, new Literal(Source.EMPTY, value, DataType.LONG)); - EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), List.of()); + EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), Map.of(), Map.of(), List.of()); Filter filter = new Filter(Source.EMPTY, esRelation, greaterThan); return new FragmentExec(filter); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index bb8d87715364e..11c74606efe66 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -303,6 +303,8 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR plan.source(), esIndex.name(), plan.indexMode(), + esIndex.originalIndices(), + esIndex.concreteIndices(), esIndex.indexNameWithModes(), attributes.isEmpty() ? NO_FIELDS : attributes ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index 8667b313f771e..a54aa219623b6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -456,9 +456,12 @@ public void messageReceived(LookupRequest request, TransportChannel channel, Tas false, false, refs.acquire(indexResult -> { - if (indexResult.isValid() && indexResult.get().concreteIndices().size() == 1) { + if (indexResult.isValid() && indexResult.get().concreteQualifiedIndices().size() == 1) { EsIndex esIndex = indexResult.get(); - var concreteIndices = Map.of(request.clusterAlias, Iterables.get(esIndex.concreteIndices(), 0)); + var concreteIndices = Map.of( + request.clusterAlias, + Iterables.get(esIndex.concreteQualifiedIndices(), 0) + ); var resolved = new ResolvedEnrichPolicy( p.getMatchField(), p.getType(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java index 51fe5b1bf137c..e070c81a81429 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java @@ -85,7 +85,7 @@ private static QueryRewriteContext queryRewriteContext(TransportActionServices s private static Set indexNames(LogicalPlan plan) { Set indexNames = new HashSet<>(); - plan.forEachDown(EsRelation.class, esRelation -> indexNames.addAll(esRelation.concreteIndices())); + plan.forEachDown(EsRelation.class, esRelation -> indexNames.addAll(esRelation.concreteQualifiedIndices())); return indexNames; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java index 97468dd1c76a8..dcd9a048a9f86 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java @@ -9,6 +9,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.type.EsField; +import java.util.List; import java.util.Map; import java.util.Set; @@ -16,6 +17,8 @@ public record EsIndex( String name, Map mapping, // keyed by field names Map indexNameWithModes, + Map> originalIndices, // keyed by cluster alias + Map> concreteIndices, // keyed by cluster alias Set partiallyUnmappedFields ) { @@ -29,7 +32,7 @@ public boolean isPartiallyUnmappedField(String fieldName) { return partiallyUnmappedFields.contains(fieldName); } - public Set concreteIndices() { + public Set concreteQualifiedIndices() { return indexNameWithModes.keySet(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java index c2e8663150339..cce15a7b83352 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java @@ -33,11 +33,11 @@ public static IndexResolution valid(EsIndex index, Set resolvedIndices, * Use this method only if the set of concrete resolved indices is the same as EsIndex#concreteIndices(). */ public static IndexResolution valid(EsIndex index) { - return valid(index, index.concreteIndices(), Map.of()); + return valid(index, index.concreteQualifiedIndices(), Map.of()); } public static IndexResolution empty(String indexPattern) { - return valid(new EsIndex(indexPattern, Map.of(), Map.of(), Set.of())); + return valid(new EsIndex(indexPattern, Map.of(), Map.of(), Map.of(), Map.of(), Set.of())); } public static IndexResolution invalid(String invalid) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnEmptyMappings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnEmptyMappings.java index 0255e8aaadffb..94461a4cbbb91 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnEmptyMappings.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnEmptyMappings.java @@ -16,6 +16,6 @@ public final class SkipQueryOnEmptyMappings extends OptimizerRules.OptimizerRule @Override protected LogicalPlan rule(EsRelation plan) { - return plan.concreteIndices().isEmpty() ? new LocalRelation(plan.source(), plan.output(), EmptyLocalSupplier.EMPTY) : plan; + return plan.concreteQualifiedIndices().isEmpty() ? new LocalRelation(plan.source(), plan.output(), EmptyLocalSupplier.EMPTY) : plan; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index 718f59604bb6e..fcbf4bf504de6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -33,6 +33,9 @@ public class EsRelation extends LeafPlan { private final String indexPattern; private final IndexMode indexMode; + // the below data structures are only needed on coordinator and are transient for now + private final transient Map> originalIndices; // keyed by cluster alias + private final transient Map> concreteIndices; // keyed by cluster alias private final Map indexNameWithModes; private final List attrs; @@ -40,12 +43,16 @@ public EsRelation( Source source, String indexPattern, IndexMode indexMode, + Map> originalIndices, + Map> concreteIndices, Map indexNameWithModes, List attributes ) { super(source); this.indexPattern = indexPattern; this.indexMode = indexMode; + this.originalIndices = originalIndices; + this.concreteIndices = concreteIndices; this.indexNameWithModes = indexNameWithModes; this.attrs = attributes; } @@ -63,7 +70,7 @@ private static EsRelation readFrom(StreamInput in) throws IOException { if (in.getTransportVersion().supports(TransportVersions.V_8_18_0) == false) { in.readBoolean(); } - return new EsRelation(source, indexPattern, indexMode, indexNameWithModes, attributes); + return new EsRelation(source, indexPattern, indexMode, Map.of(), Map.of(), indexNameWithModes, attributes); } @Override @@ -89,7 +96,7 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, EsRelation::new, indexPattern, indexMode, indexNameWithModes, attrs); + return NodeInfo.create(this, EsRelation::new, indexPattern, indexMode, originalIndices, concreteIndices, indexNameWithModes, attrs); } public String indexPattern() { @@ -100,6 +107,14 @@ public IndexMode indexMode() { return indexMode; } + public Map> originalIndices() { + return originalIndices; + } + + public Map> concreteIndices() { + return concreteIndices; + } + public Map indexNameWithModes() { return indexNameWithModes; } @@ -109,7 +124,7 @@ public List output() { return attrs; } - public Set concreteIndices() { + public Set concreteQualifiedIndices() { return indexNameWithModes.keySet(); } @@ -122,7 +137,7 @@ public boolean expressionsResolved() { @Override public int hashCode() { - return Objects.hash(indexPattern, indexMode, indexNameWithModes, attrs); + return Objects.hash(indexPattern, indexMode, originalIndices, concreteIndices, indexNameWithModes, attrs); } @Override @@ -138,6 +153,8 @@ public boolean equals(Object obj) { EsRelation other = (EsRelation) obj; return Objects.equals(indexPattern, other.indexPattern) && Objects.equals(indexMode, other.indexMode) + && Objects.equals(originalIndices, other.originalIndices) + && Objects.equals(concreteIndices, other.concreteIndices) && Objects.equals(indexNameWithModes, other.indexNameWithModes) && Objects.equals(attrs, other.attrs); } @@ -153,10 +170,10 @@ public String nodeString() { } public EsRelation withAttributes(List newAttributes) { - return new EsRelation(source(), indexPattern, indexMode, indexNameWithModes, newAttributes); + return new EsRelation(source(), indexPattern, indexMode, originalIndices, concreteIndices, indexNameWithModes, newAttributes); } public EsRelation withIndexMode(IndexMode indexMode) { - return new EsRelation(source(), indexPattern, indexMode, indexNameWithModes, attrs); + return new EsRelation(source(), indexPattern, indexMode, originalIndices, concreteIndices, indexNameWithModes, attrs); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 41ae7a1b5bc7f..09a0c388c6c5a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -163,7 +163,7 @@ public static Set planConcreteIndices(PhysicalPlan plan) { return Set.of(); } var indices = new LinkedHashSet(); - forEachRelation(plan, relation -> indices.addAll(relation.concreteIndices())); + forEachRelation(plan, relation -> indices.addAll(relation.concreteQualifiedIndices())); return indices; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index f95220ac1007f..8728010a75433 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -760,8 +760,15 @@ private IndexResolution checkSingleIndex( var localIndexNames = indexNames.stream().map(n -> RemoteClusterAware.splitIndexName(n)[1]).collect(toSet()); if (localIndexNames.size() == 1) { String indexName = localIndexNames.iterator().next(); - EsIndex newIndex = new EsIndex(index, lookupIndexResolution.get().mapping(), Map.of(indexName, IndexMode.LOOKUP), Set.of()); - return IndexResolution.valid(newIndex, newIndex.concreteIndices(), lookupIndexResolution.failures()); + EsIndex newIndex = new EsIndex( + index, + lookupIndexResolution.get().mapping(), + Map.of(indexName, IndexMode.LOOKUP), + Map.of(), + Map.of(), + Set.of() + ); + return IndexResolution.valid(newIndex, newIndex.concreteQualifiedIndices(), lookupIndexResolution.failures()); } // validate remotes to be able to handle multiple indices in LOOKUP JOIN validateRemoteVersions(executionInfo); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 26d9f67925a33..eb33cdffb9299 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -272,7 +272,14 @@ public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fie // mapping index will generate no columns (important) for a query like FROM empty-mapping-index, whereas an empty result here but // for fields that do not exist in the index (but the index has a mapping) will result in "VerificationException Unknown column" // errors. - var index = new EsIndex(indexPattern, rootFields, allEmpty ? Map.of() : concreteIndices, partiallyUnmappedFields); + var index = new EsIndex( + indexPattern, + rootFields, + allEmpty ? Map.of() : concreteIndices, + Map.of(), + Map.of(), + partiallyUnmappedFields + ); var failures = EsqlCCSUtils.groupFailuresPerCluster(fieldsInfo.caps.getFailures()); return IndexResolution.valid(index, concreteIndices.keySet(), failures); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 8841be5023343..0867375f5de5c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -452,7 +452,14 @@ private static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndex .toList(); var mergedMappings = mergeMappings(mappings); return IndexResolution.valid( - new EsIndex(datasets.indexPattern(), mergedMappings.mapping, indexModes, mergedMappings.partiallyUnmappedFields) + new EsIndex( + datasets.indexPattern(), + mergedMappings.mapping, + indexModes, + Map.of(), + Map.of(), + mergedMappings.partiallyUnmappedFields + ) ); } @@ -535,7 +542,10 @@ private static EnrichResolution loadEnrichPolicies() { // this could practically work, but it's wrong: // EnrichPolicyResolution should contain the policy (system) index, not the source index EsIndex esIndex = loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset.of(sourceIndex.withTypeMapping(Map.of()))).get(); - var concreteIndices = Map.of(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, Iterables.get(esIndex.concreteIndices(), 0)); + var concreteIndices = Map.of( + RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, + Iterables.get(esIndex.concreteQualifiedIndices(), 0) + ); enrichResolution.addResolvedPolicy( policyConfig.policyName(), Enrich.Mode.ANY, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index 3325076799204..354c38267546a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -203,7 +203,9 @@ public static UnresolvedRelation unresolvedRelation(String index) { } public static IndexResolution loadMapping(String resource, String indexName, IndexMode indexMode) { - return IndexResolution.valid(new EsIndex(indexName, EsqlTestUtils.loadMapping(resource), Map.of(indexName, indexMode), Set.of())); + return IndexResolution.valid( + new EsIndex(indexName, EsqlTestUtils.loadMapping(resource), Map.of(indexName, indexMode), Map.of(), Map.of(), Set.of()) + ); } public static IndexResolution loadMapping(String resource, String indexName) { @@ -408,6 +410,8 @@ public static IndexResolution indexWithDateDateNanosUnionType() { "index*", Map.of(dateDateNanos, dateDateNanosField, dateDateNanosLong, dateDateNanosLongField), Map.of("index1", IndexMode.STANDARD, "index2", IndexMode.STANDARD, "index3", IndexMode.STANDARD), + Map.of(), + Map.of(), Set.of() ); return IndexResolution.valid(index); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 942e5eb132d37..a45ec64af3ecd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -195,7 +195,10 @@ public void testIndexResolution() { var plan = analyzer.analyze(UNRESOLVED_RELATION); var limit = as(plan, Limit.class); - assertEquals(new EsRelation(EMPTY, idx.name(), IndexMode.STANDARD, idx.indexNameWithModes(), NO_FIELDS), limit.child()); + assertEquals( + new EsRelation(EMPTY, idx.name(), IndexMode.STANDARD, Map.of(), Map.of(), idx.indexNameWithModes(), NO_FIELDS), + limit.child() + ); } public void testFailOnUnresolvedIndex() { @@ -213,7 +216,10 @@ public void testIndexWithClusterResolution() { var plan = analyzer.analyze(unresolvedRelation("cluster:idx")); var limit = as(plan, Limit.class); - assertEquals(new EsRelation(EMPTY, idx.name(), IndexMode.STANDARD, idx.indexNameWithModes(), NO_FIELDS), limit.child()); + assertEquals( + new EsRelation(EMPTY, idx.name(), IndexMode.STANDARD, Map.of(), Map.of(), idx.indexNameWithModes(), NO_FIELDS), + limit.child() + ); } public void testAttributeResolution() { @@ -4705,6 +4711,8 @@ public void testImplicitCastingForAggregateMetricDouble() { "k8s*", mapping, Map.of("k8s", IndexMode.TIME_SERIES, "k8s-downsampled", IndexMode.TIME_SERIES), + Map.of(), + Map.of(), Set.of() ); var analyzer = new Analyzer( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java index e02ef6110e5c4..bd1e40917cbad 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java @@ -300,7 +300,7 @@ private FragmentExec buildLessThanFilter(int value) { new EsField("lint", DataType.INTEGER, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) ); Expression lessThan = new LessThan(Source.EMPTY, filterAttribute, new Literal(Source.EMPTY, value, DataType.INTEGER)); - EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), List.of()); + EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), Map.of(), Map.of(), List.of()); Filter filter = new Filter(Source.EMPTY, esRelation, lessThan); return new FragmentExec(filter); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java index 2efbf4f6c5911..9e10fa2eb2cc9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java @@ -13,39 +13,48 @@ import org.elasticsearch.xpack.esql.type.EsFieldTests; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import static org.elasticsearch.core.Tuple.tuple; +import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.ESTestCase.randomIdentifier; +import static org.elasticsearch.test.ESTestCase.randomList; +import static org.elasticsearch.test.ESTestCase.randomMap; public class EsIndexGenerator { public static EsIndex esIndex(String name) { - return new EsIndex(name, Map.of(), Map.of(), Set.of()); + return new EsIndex(name, Map.of(), Map.of(), Map.of(), Map.of(), Set.of()); } public static EsIndex esIndex(String name, Map mapping) { - return new EsIndex(name, mapping, Map.of(), Set.of()); + return new EsIndex(name, mapping, Map.of(), Map.of(), Map.of(), Set.of()); } public static EsIndex esIndex(String name, Map mapping, Map indexNameWithModes) { - return new EsIndex(name, mapping, indexNameWithModes, Set.of()); + return new EsIndex(name, mapping, indexNameWithModes, Map.of(), Map.of(), Set.of()); } public static EsIndex randomEsIndex() { - return new EsIndex(ESTestCase.randomIdentifier(), randomMapping(), randomIndexNameWithModes(), Set.of()); + return new EsIndex(randomIdentifier(), randomMapping(), randomIndexNameWithModes(), Map.of(), Map.of(), Set.of()); } public static Map randomMapping() { int size = ESTestCase.between(0, 10); Map result = new HashMap<>(size); while (result.size() < size) { - result.put(ESTestCase.randomIdentifier(), EsFieldTests.randomAnyEsField(1)); + result.put(randomIdentifier(), EsFieldTests.randomAnyEsField(1)); } return result; } public static Map randomIndexNameWithModes() { - return ESTestCase.randomMap(0, 10, () -> tuple(ESTestCase.randomIdentifier(), ESTestCase.randomFrom(IndexMode.values()))); + return randomMap(0, 10, () -> tuple(randomIdentifier(), randomFrom(IndexMode.values()))); + } + + public static Map> randomRemotesWithIndices() { + return randomMap(0, 10, () -> tuple(randomIdentifier(), randomList(1, 10, ESTestCase::randomIdentifier))); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java index a3afd198f736e..cc74092ac5578 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java @@ -191,6 +191,8 @@ public static void init() { "multi_index", multiIndexMapping, Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), Set.of("partial_type_keyword") ); multiIndexAnalyzer = new Analyzer( @@ -205,7 +207,14 @@ public static void init() { ); var sampleDataMapping = loadMapping("mapping-sample_data.json"); - var sampleDataIndex = new EsIndex("sample_data", sampleDataMapping, Map.of("sample_data", IndexMode.STANDARD), Set.of()); + var sampleDataIndex = new EsIndex( + "sample_data", + sampleDataMapping, + Map.of("sample_data", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of() + ); sampleDataIndexAnalyzer = new Analyzer( testAnalyzerContext( EsqlTestUtils.TEST_CFG, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index ccf2a923efc7f..0795856880450 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -6160,9 +6160,9 @@ public void testInlineStatsWithLookupJoin() { var join = as(topN.child(), Join.class); assertThat(Expressions.names(join.config().leftFields()), is(List.of("scalerank"))); var left = as(join.left(), EsRelation.class); - assertThat(left.concreteIndices(), is(Set.of("airports"))); + assertThat(left.concreteQualifiedIndices(), is(Set.of("airports"))); var right = as(join.right(), EsRelation.class); - assertThat(right.concreteIndices(), is(Set.of("languages_lookup"))); + assertThat(right.concreteQualifiedIndices(), is(Set.of("languages_lookup"))); } /* diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index f30392bc3a403..8984bc73188ef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -3250,6 +3250,8 @@ public void testProjectAwayColumns() { Source.EMPTY, index.name(), IndexMode.STANDARD, + Map.of(), + Map.of(), index.indexNameWithModes(), esField.stream().map(field -> (Attribute) new FieldAttribute(Source.EMPTY, null, null, field.getName(), field)).toList() ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java index 7ff02164961e5..45b4e04449951 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/promql/PromqlLogicalPlanOptimizerTests.java @@ -66,7 +66,9 @@ public static void initTest() { assumeTrue("requires snapshot build with promql feature enabled", PromqlFeatures.isEnabled()); var timeSeriesMapping = loadMapping("k8s-mappings.json"); - var timeSeriesIndex = IndexResolution.valid(new EsIndex("k8s", timeSeriesMapping, Map.of("k8s", IndexMode.TIME_SERIES), Set.of())); + var timeSeriesIndex = IndexResolution.valid( + new EsIndex("k8s", timeSeriesMapping, Map.of("k8s", IndexMode.TIME_SERIES), Map.of(), Map.of(), Set.of()) + ); tsAnalyzer = new Analyzer( new AnalyzerContext( EsqlTestUtils.TEST_CFG, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java index 120254edd884b..96155b2b18ad2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java @@ -393,7 +393,7 @@ private static EsRelation relation() { } private static EsRelation relation(List fieldAttributes) { - return new EsRelation(EMPTY, randomIdentifier(), randomFrom(IndexMode.values()), Map.of(), fieldAttributes); + return new EsRelation(EMPTY, randomIdentifier(), randomFrom(IndexMode.values()), Map.of(), Map.of(), Map.of(), fieldAttributes); } public void testPushDownFilterPastLeftJoinWithPushable() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EsRelationSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EsRelationSerializationTests.java index 0a0534c3e86f4..878c069977d83 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EsRelationSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EsRelationSerializationTests.java @@ -17,6 +17,7 @@ import java.util.Map; import static org.elasticsearch.xpack.esql.index.EsIndexGenerator.randomIndexNameWithModes; +import static org.elasticsearch.xpack.esql.index.EsIndexGenerator.randomRemotesWithIndices; public class EsRelationSerializationTests extends AbstractLogicalPlanSerializationTests { public static EsRelation randomEsRelation() { @@ -24,6 +25,8 @@ public static EsRelation randomEsRelation() { randomSource(), randomIdentifier(), randomFrom(IndexMode.values()), + randomRemotesWithIndices(), + randomRemotesWithIndices(), randomIndexNameWithModes(), randomFieldAttributes(0, 10, false) ); @@ -38,16 +41,20 @@ protected EsRelation createTestInstance() { protected EsRelation mutateInstance(EsRelation instance) throws IOException { String indexPattern = instance.indexPattern(); IndexMode indexMode = instance.indexMode(); + Map> originalIndices = instance.originalIndices(); + Map> concreteIndices = instance.concreteIndices(); Map indexNameWithModes = instance.indexNameWithModes(); List attributes = instance.output(); - switch (between(0, 3)) { + switch (between(0, 5)) { case 0 -> indexPattern = randomValueOtherThan(indexPattern, ESTestCase::randomIdentifier); case 1 -> indexMode = randomValueOtherThan(indexMode, () -> randomFrom(IndexMode.values())); case 2 -> indexNameWithModes = randomValueOtherThan(indexNameWithModes, EsIndexGenerator::randomIndexNameWithModes); - case 3 -> attributes = randomValueOtherThan(attributes, () -> randomFieldAttributes(0, 10, false)); + case 3 -> originalIndices = randomValueOtherThan(originalIndices, EsIndexGenerator::randomRemotesWithIndices); + case 4 -> concreteIndices = randomValueOtherThan(concreteIndices, EsIndexGenerator::randomRemotesWithIndices); + case 5 -> attributes = randomValueOtherThan(attributes, () -> randomFieldAttributes(0, 10, false)); default -> throw new IllegalArgumentException(); } - return new EsRelation(instance.source(), indexPattern, indexMode, indexNameWithModes, attributes); + return new EsRelation(instance.source(), indexPattern, indexMode, originalIndices, concreteIndices, indexNameWithModes, attributes); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java index c6fc0934cb469..de354ddc6e11a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java @@ -211,7 +211,15 @@ private void testSerializePlanWithIndex(EsIndex index, ByteSizeValue expected) t private void testSerializePlanWithIndex(EsIndex index, ByteSizeValue expected, boolean keepAllFields) throws IOException { List allAttributes = Analyzer.mappingAsAttributes(randomSource(), index.mapping()); List keepAttributes = keepAllFields || allAttributes.isEmpty() ? allAttributes : List.of(allAttributes.getFirst()); - EsRelation relation = new EsRelation(randomSource(), index.name(), IndexMode.STANDARD, index.indexNameWithModes(), keepAttributes); + EsRelation relation = new EsRelation( + randomSource(), + index.name(), + IndexMode.STANDARD, + Map.of(), + Map.of(), + index.indexNameWithModes(), + keepAttributes + ); Limit limit = new Limit(randomSource(), new Literal(randomSource(), 10, DataType.INTEGER), relation); Project project = new Project(randomSource(), limit, limit.output()); FragmentExec fragmentExec = new FragmentExec(project); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtilsTests.java index 4daa21a38f984..11e64300009cd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtilsTests.java @@ -223,7 +223,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { ) ); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of()); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteQualifiedIndices(), Map.of()); EsqlCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, Set.of(indexResolution)); @@ -266,7 +266,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { IndexMode.STANDARD ) ); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of()); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteQualifiedIndices(), Map.of()); EsqlCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, Set.of(indexResolution)); @@ -308,7 +308,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { // remote1 is unavailable var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); var failures = Map.of(REMOTE1_ALIAS, List.of(failure)); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), failures); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteQualifiedIndices(), failures); EsqlCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, Set.of(indexResolution)); @@ -350,7 +350,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); var failures = Map.of(REMOTE1_ALIAS, List.of(failure)); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), failures); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteQualifiedIndices(), failures); EsqlCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, Set.of(indexResolution)); EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER_ALIAS); @@ -398,7 +398,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { // remote1 is unavailable var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); var failures = Map.of(REMOTE1_ALIAS, List.of(failure)); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), failures); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteQualifiedIndices(), failures); EsqlCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, Set.of(indexResolution)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index b5140896fbbb8..c0bbd51a00908 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -99,6 +99,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.esql.index.EsIndexGenerator.randomEsIndex; import static org.elasticsearch.xpack.esql.index.EsIndexGenerator.randomIndexNameWithModes; +import static org.elasticsearch.xpack.esql.index.EsIndexGenerator.randomRemotesWithIndices; import static org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests.randomFieldAttributes; import static org.elasticsearch.xpack.esql.plan.physical.LookupJoinExecSerializationTests.randomJoinOnExpression; import static org.mockito.Mockito.mock; @@ -733,6 +734,8 @@ static EsRelation randomEsRelation() { SourceTests.randomSource(), randomIdentifier(), randomFrom(IndexMode.values()), + randomRemotesWithIndices(), + randomRemotesWithIndices(), randomIndexNameWithModes(), randomFieldAttributes(0, 10, false) ); From 2a92340ddbecda90dc3a877083d55b11fe956d6a Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 20 Nov 2025 16:42:43 +0100 Subject: [PATCH 2/7] populate new data structure with concrete indices --- .../transport/RemoteClusterAware.java | 21 +++++++++++++------ .../xpack/esql/session/IndexResolver.java | 19 ++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 86f15184ece21..508fefe2c764d 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -88,12 +88,21 @@ public static List getRemoteIndexExpressions(String... expressions) { */ public static String parseClusterAlias(String indexExpression) { assert indexExpression != null : "Must not pass null indexExpression"; - String[] parts = splitIndexName(indexExpression.trim()); - if (parts[0] == null) { - return RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - } else { - return parts[0]; - } + return getClusterAlias(splitIndexName(indexExpression.trim())); + } + + /** + * @return the cluster alias or LOCAL_CLUSTER_GROUP_KEY if the split represents local index + */ + public static String getClusterAlias(String[] split) { + return split[0] == null ? LOCAL_CLUSTER_GROUP_KEY : split[0]; + } + + /** + * @return the local index name from the qualified index name split by RemoteClusterAware#splitIndexName + */ + public static String getLocalIndexName(String[] split) { + return split[1]; } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index eb33cdffb9299..bf81befc69e81 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -24,6 +24,7 @@ import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsResponse; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; @@ -257,15 +258,17 @@ public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fie } } - Map concreteIndices = Maps.newMapWithExpectedSize(fieldsInfo.caps.getIndexResponses().size()); - for (FieldCapabilitiesIndexResponse ir : fieldsInfo.caps.getIndexResponses()) { - concreteIndices.put(ir.getIndexName(), ir.getIndexMode()); - } - boolean allEmpty = true; + Map indexNameWithModes = Maps.newMapWithExpectedSize(fieldsInfo.caps.getIndexResponses().size()); + Map> concreteIndices = Maps.newHashMapWithExpectedSize(8); for (FieldCapabilitiesIndexResponse ir : fieldsInfo.caps.getIndexResponses()) { allEmpty &= ir.get().isEmpty(); + indexNameWithModes.put(ir.getIndexName(), ir.getIndexMode()); + var parts = RemoteClusterAware.splitIndexName(ir.getIndexName()); + concreteIndices.computeIfAbsent(RemoteClusterAware.getClusterAlias(parts), k -> new ArrayList<>()) + .add(RemoteClusterAware.getLocalIndexName(parts)); } + // If all the mappings are empty we return an empty set of resolved indices to line up with QL // Introduced with #46775 // We need to be able to differentiate between an empty mapping index and an empty index due to fields not being found. An empty @@ -275,13 +278,13 @@ public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fie var index = new EsIndex( indexPattern, rootFields, - allEmpty ? Map.of() : concreteIndices, - Map.of(), + allEmpty ? Map.of() : indexNameWithModes, Map.of(), + concreteIndices, partiallyUnmappedFields ); var failures = EsqlCCSUtils.groupFailuresPerCluster(fieldsInfo.caps.getFailures()); - return IndexResolution.valid(index, concreteIndices.keySet(), failures); + return IndexResolution.valid(index, indexNameWithModes.keySet(), failures); } private record IndexFieldCapabilitiesWithSourceHash(List fieldCapabilities, String indexMappingHash) {} From 68fac4be3c655daa2c9fee67705565885e0a0168 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 21 Nov 2025 10:45:09 +0100 Subject: [PATCH 3/7] serialized stored indices. --- .../esql_es_relation_add_split_indices.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../xpack/esql/plan/logical/EsRelation.java | 24 +++++++++++++++---- .../esql/plan/physical/EsSourceExec.java | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/esql_es_relation_add_split_indices.csv diff --git a/server/src/main/resources/transport/definitions/referable/esql_es_relation_add_split_indices.csv b/server/src/main/resources/transport/definitions/referable/esql_es_relation_add_split_indices.csv new file mode 100644 index 0000000000000..5942497c1d850 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_es_relation_add_split_indices.csv @@ -0,0 +1 @@ +9226000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 6c7029b08b818..484d263bd3a08 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -keystore_details_in_reload_secure_settings_response,9225000 +esql_es_relation_add_split_indices,9226000 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index fcbf4bf504de6..9be40b81785f8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.plan.logical; +import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -25,6 +26,9 @@ import java.util.Set; public class EsRelation extends LeafPlan { + + private static final TransportVersion SPLIT_INDICES = TransportVersion.fromName("esql_es_relation_add_split_indices"); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( LogicalPlan.class, "EsRelation", @@ -33,9 +37,8 @@ public class EsRelation extends LeafPlan { private final String indexPattern; private final IndexMode indexMode; - // the below data structures are only needed on coordinator and are transient for now - private final transient Map> originalIndices; // keyed by cluster alias - private final transient Map> concreteIndices; // keyed by cluster alias + private final Map> originalIndices; // keyed by cluster alias + private final Map> concreteIndices; // keyed by cluster alias private final Map indexNameWithModes; private final List attrs; @@ -64,13 +67,22 @@ private static EsRelation readFrom(StreamInput in) throws IOException { // this used to be part of EsIndex deserialization in.readImmutableMap(StreamInput::readString, EsField::readFrom); } + Map> originalIndices; + Map> concreteIndices; + if (in.getTransportVersion().supports(SPLIT_INDICES)) { + originalIndices = in.readMapOfLists(StreamInput::readString); + concreteIndices = in.readMapOfLists(StreamInput::readString); + } else { + originalIndices = Map.of(); + concreteIndices = Map.of(); + } Map indexNameWithModes = in.readMap(IndexMode::readFrom); List attributes = in.readNamedWriteableCollectionAsList(Attribute.class); IndexMode indexMode = IndexMode.fromString(in.readString()); if (in.getTransportVersion().supports(TransportVersions.V_8_18_0) == false) { in.readBoolean(); } - return new EsRelation(source, indexPattern, indexMode, Map.of(), Map.of(), indexNameWithModes, attributes); + return new EsRelation(source, indexPattern, indexMode, originalIndices, concreteIndices, indexNameWithModes, attributes); } @Override @@ -81,6 +93,10 @@ public void writeTo(StreamOutput out) throws IOException { // this used to be part of EsIndex serialization out.writeMap(Map.of(), (o, x) -> x.writeTo(out)); } + if (out.getTransportVersion().supports(SPLIT_INDICES)) { + out.writeMap(originalIndices, StreamOutput::writeStringCollection); + out.writeMap(concreteIndices, StreamOutput::writeStringCollection); + } out.writeMap(indexNameWithModes, (o, v) -> IndexMode.writeTo(v, out)); out.writeNamedWriteableCollection(attrs); out.writeString(indexMode.getName()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java index b6226c82b9c20..b1ebe8b43887a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java @@ -29,7 +29,7 @@ public class EsSourceExec extends LeafExec { - protected static final TransportVersion REMOVE_NAME_WITH_MODS = TransportVersion.fromName("esql_es_source_remove_name_with_mods"); + private static final TransportVersion REMOVE_NAME_WITH_MODS = TransportVersion.fromName("esql_es_source_remove_name_with_mods"); public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( PhysicalPlan.class, From f5cdcef39a6c8ae24d82898bdcf5a6c782b13ebd Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 21 Nov 2025 13:32:11 +0100 Subject: [PATCH 4/7] fox various tests --- .../NodesReloadSecureSettingsResponse.java | 2 +- ...NodesReloadSecureSettingsResponseTests.java | 2 +- .../ExchangeSinkExecSerializationTests.java | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java index 873c67bf45cb5..d6ba1d7ce850c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java @@ -92,7 +92,7 @@ public String toString() { public static class NodeResponse extends BaseNodeResponse { - private static final TransportVersion KEYSTORE_DETAILS = TransportVersion.fromName( + public static final TransportVersion KEYSTORE_DETAILS = TransportVersion.fromName( "keystore_details_in_reload_secure_settings_response" ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponseTests.java index 91d87de99a00f..9209a4d9d9197 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponseTests.java @@ -98,7 +98,7 @@ private void verifyWriteAndRead(NodesReloadSecureSettingsResponse.NodeResponse n assertNotNull(readNr.reloadException()); assertEquals(nr.reloadException().getMessage(), readNr.reloadException().getMessage()); } - if (version.equals(TransportVersion.current())) { + if (version.supports(NodesReloadSecureSettingsResponse.NodeResponse.KEYSTORE_DETAILS)) { assertArrayEquals(nr.secureSettingNames(), readNr.secureSettingNames()); assertEquals(nr.keystorePath(), readNr.keystorePath()); assertEquals(nr.keystoreDigest(), readNr.keystoreDigest()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java index de354ddc6e11a..aa7b3f1152d6d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java @@ -87,8 +87,9 @@ public void testManyTypeConflicts() throws IOException { * 1019093b - remove unused fields from FieldAttribute #127854 * 1026343b - added time series field type to EsField #129649 * 1033593b - added qualifier back to FieldAttribute #132925 + * 1033595b - added split indices to EsRelation #138396 */ - testManyTypeConflicts(false, ByteSizeValue.ofBytes(1033593)); + testManyTypeConflicts(false, ByteSizeValue.ofBytes(1033595)); } /** @@ -109,8 +110,9 @@ public void testManyTypeConflictsWithParent() throws IOException { * 1964273b - remove unused fields from FieldAttribute #127854 * 1971523b - added time series field type to EsField #129649 * 1986023b - added qualifier back to FieldAttribute #132925 + * 1986025b - added split indices to EsRelation #138396 */ - testManyTypeConflicts(true, ByteSizeValue.ofBytes(1986023)); + testManyTypeConflicts(true, ByteSizeValue.ofBytes(1986025)); } private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) throws IOException { @@ -133,13 +135,14 @@ public void testDeeplyNestedFields() throws IOException { * 43402881b - remove unused fields from FieldAttribute #127854 * 43665025b - added time series field type to EsField #129649 * 43927169b - added qualifier back to FieldAttribute #132925 + * 43927171b - added split indices to EsRelation #138396 */ int depth = 6; int childrenPerLevel = 8; EsIndex index = deeplyNestedIndex(depth, childrenPerLevel); - testSerializePlanWithIndex(index, ByteSizeValue.ofBytes(43927169L)); + testSerializePlanWithIndex(index, ByteSizeValue.ofBytes(43927171L)); } /** @@ -157,13 +160,14 @@ public void testDeeplyNestedFieldsKeepOnlyOne() throws IOException { * 350b - remove unused fields from FieldAttribute #127854 * 351b - added time series field type to EsField #129649 * 352b - added qualifier back to FieldAttribute #132925 + * 354b - added split indices to EsRelation #138396 */ int depth = 6; int childrenPerLevel = 9; EsIndex index = deeplyNestedIndex(depth, childrenPerLevel); - testSerializePlanWithIndex(index, ByteSizeValue.ofBytes(352), false); + testSerializePlanWithIndex(index, ByteSizeValue.ofBytes(354), false); } /** @@ -173,7 +177,9 @@ public void testDeeplyNestedFieldsKeepOnlyOne() throws IOException { */ public void testIndexPatternTargetingMultipleIndices() throws IOException { /* - * History: 4996b - initial + * History: + * 4996b - initial + * 4998b - added split indices to EsRelation #138396 */ var index = EsIndexGenerator.esIndex( @@ -183,7 +189,7 @@ public void testIndexPatternTargetingMultipleIndices() throws IOException { .mapToObj(i -> "partial-.ds-index-service-logs-2025.01.01-000" + i) .collect(toMap(Function.identity(), i -> IndexMode.STANDARD)) ); - testSerializePlanWithIndex(index, ByteSizeValue.ofBytes(4996)); + testSerializePlanWithIndex(index, ByteSizeValue.ofBytes(4998)); } /** From 3788e2c80ac1a4ea314e0b46db4e71c97e71d6bd Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 21 Nov 2025 14:23:01 +0100 Subject: [PATCH 5/7] use split concrete indices --- .../xpack/esql/core/util/Holder.java | 15 +++++++ .../xpack/esql/plugin/IndexResolutionIT.java | 45 +++++++++---------- .../xpack/esql/planner/PlannerUtils.java | 14 +----- .../xpack/esql/plugin/ComputeService.java | 19 ++++++-- 4 files changed, 53 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Holder.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Holder.java index 9aadcefb84e84..a248b5aff2a48 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Holder.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Holder.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.core.util; +import java.util.function.Supplier; + /** * Simply utility class used for setting a state, typically * for closures (which require outside variables to be final). @@ -36,7 +38,20 @@ public void setIfAbsent(T value) { } } + /** + * Sets a value in the holder, but only if none has already been set. + * @param value the new value to set. + */ + public void setOnce(T value) { + assert this.value == null : "Value has already been set to " + this.value; + this.value = value; + } + public T get() { return value; } + + public T getOrDefault(Supplier defaultValue) { + return value != null ? value : defaultValue.get(); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java index a5fcbcb9f1ca1..50534d2a4bab7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.datastreams.DataStreamsPlugin; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.plugins.Plugin; @@ -232,28 +231,28 @@ public void testPartialResolution() { assertAcked(client().admin().indices().prepareCreate("index-1")); indexRandom(true, "index-1", 1); - expectThrows( - IndexNotFoundException.class, - equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty - () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1")) - ); - expectThrows( - IndexNotFoundException.class, - equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty even if allow_partial=true - () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1").allowPartialResults(true)) - ); - expectThrows( - IndexNotFoundException.class, - equalTo("no such index [nonexisting-1]"), // only the first missing index is reported - () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1,nonexisting-2")) - ); - - assertAcked(client().admin().indices().prepareCreate("index-2").setMapping("field", "type=keyword")); - expectThrows( - IndexNotFoundException.class, - equalTo("no such index [nonexisting-1]"), // fails when present index has no documents and non-empty mapping - () -> run(syncEsqlQueryRequest().query("FROM index-2,nonexisting-1")) - ); + // expectThrows( + // IndexNotFoundException.class, + // equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty + // () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1")) + // ); + // expectThrows( + // IndexNotFoundException.class, + // equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty even if allow_partial=true + // () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1").allowPartialResults(true)) + // ); + // expectThrows( + // IndexNotFoundException.class, + // equalTo("no such index [nonexisting-1]"), // only the first missing index is reported + // () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1,nonexisting-2")) + // ); + // + // assertAcked(client().admin().indices().prepareCreate("index-2").setMapping("field", "type=keyword")); + // expectThrows( + // IndexNotFoundException.class, + // equalTo("no such index [nonexisting-1]"), // fails when present index has no documents and non-empty mapping + // () -> run(syncEsqlQueryRequest().query("FROM index-2,nonexisting-1")) + // ); assertAcked(client().admin().indices().prepareCreate("index-3")); try (var response = run(syncEsqlQueryRequest().query("FROM index-3,nonexisting-1"))) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 09a0c388c6c5a..c457415783469 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -155,18 +155,6 @@ private static ReducedPlan getPhysicalPlanReduction(int estimatedRowSize, Physic return new ReducedPlan(EstimatesRowSize.estimateRowSize(estimatedRowSize, plan)); } - /** - * Returns a set of concrete indices after resolving the original indices specified in the FROM command. - */ - public static Set planConcreteIndices(PhysicalPlan plan) { - if (plan == null) { - return Set.of(); - } - var indices = new LinkedHashSet(); - forEachRelation(plan, relation -> indices.addAll(relation.concreteQualifiedIndices())); - return indices; - } - /** * Returns the original indices specified in the FROM command of the query. We need the original query to resolve alias filters. */ @@ -188,7 +176,7 @@ public static boolean requiresSortedTimeSeriesSource(PhysicalPlan plan) { }); } - private static void forEachRelation(PhysicalPlan plan, Consumer action) { + public static void forEachRelation(PhysicalPlan plan, Consumer action) { plan.forEachDown(FragmentExec.class, f -> f.fragment().forEachDown(EsRelation.class, r -> { if (r.indexMode() != IndexMode.LOOKUP) { action.accept(r); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 1fc52270a60d2..9d270d830b2c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.Page; @@ -49,10 +50,12 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService; import org.elasticsearch.xpack.esql.inference.InferenceService; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeSourceExec; import org.elasticsearch.xpack.esql.plan.physical.OutputExec; @@ -325,9 +328,7 @@ public void executePlan( listener.onFailure(new IllegalStateException("expected data node plan starts with an ExchangeSink; got " + dataNodePlan)); return; } - Map clusterToConcreteIndices = transportService.getRemoteClusterService() - .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, PlannerUtils.planConcreteIndices(physicalPlan).toArray(String[]::new)); - QueryPragmas queryPragmas = configuration.pragmas(); + Map clusterToConcreteIndices = getIndices(physicalPlan, EsRelation::concreteIndices); Runnable cancelQueryOnFailure = cancelQueryOnFailure(rootTask); if (dataNodePlan == null) { if (clusterToConcreteIndices.values().stream().allMatch(v -> v.indices().length == 0) == false) { @@ -380,7 +381,7 @@ public void executePlan( */ List outputAttributes = physicalPlan.output(); var exchangeSource = new ExchangeSourceHandler( - queryPragmas.exchangeBufferSize(), + configuration.pragmas().exchangeBufferSize(), transportService.getThreadPool().executor(ThreadPool.Names.SEARCH) ); listener = ActionListener.runBefore(listener, () -> exchangeService.removeExchangeSourceHandler(sessionId)); @@ -799,4 +800,14 @@ public String getDescription() { return "group [" + parentDescription.get() + "]"; } } + + private static Map getIndices(PhysicalPlan plan, Function>> getter) { + var holder = new Holder>(); + PlannerUtils.forEachRelation(plan, esRelation -> { + holder.set(Maps.transformValues(getter.apply(esRelation), v -> { + return new OriginalIndices(v.toArray(String[]::new), SearchRequest.DEFAULT_INDICES_OPTIONS); + })); + }); + return holder.getOrDefault(Map::of); + } } From 808b6f1ab273af23d8a023b86477c1c63f5e7769 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 21 Nov 2025 16:44:16 +0100 Subject: [PATCH 6/7] upd --- .../xpack/esql/plugin/IndexResolutionIT.java | 45 ++++++++++--------- .../xpack/esql/session/IndexResolver.java | 3 +- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java index 50534d2a4bab7..a5fcbcb9f1ca1 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.plugins.Plugin; @@ -231,28 +232,28 @@ public void testPartialResolution() { assertAcked(client().admin().indices().prepareCreate("index-1")); indexRandom(true, "index-1", 1); - // expectThrows( - // IndexNotFoundException.class, - // equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty - // () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1")) - // ); - // expectThrows( - // IndexNotFoundException.class, - // equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty even if allow_partial=true - // () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1").allowPartialResults(true)) - // ); - // expectThrows( - // IndexNotFoundException.class, - // equalTo("no such index [nonexisting-1]"), // only the first missing index is reported - // () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1,nonexisting-2")) - // ); - // - // assertAcked(client().admin().indices().prepareCreate("index-2").setMapping("field", "type=keyword")); - // expectThrows( - // IndexNotFoundException.class, - // equalTo("no such index [nonexisting-1]"), // fails when present index has no documents and non-empty mapping - // () -> run(syncEsqlQueryRequest().query("FROM index-2,nonexisting-1")) - // ); + expectThrows( + IndexNotFoundException.class, + equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty + () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1")) + ); + expectThrows( + IndexNotFoundException.class, + equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty even if allow_partial=true + () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1").allowPartialResults(true)) + ); + expectThrows( + IndexNotFoundException.class, + equalTo("no such index [nonexisting-1]"), // only the first missing index is reported + () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1,nonexisting-2")) + ); + + assertAcked(client().admin().indices().prepareCreate("index-2").setMapping("field", "type=keyword")); + expectThrows( + IndexNotFoundException.class, + equalTo("no such index [nonexisting-1]"), // fails when present index has no documents and non-empty mapping + () -> run(syncEsqlQueryRequest().query("FROM index-2,nonexisting-1")) + ); assertAcked(client().admin().indices().prepareCreate("index-3")); try (var response = run(syncEsqlQueryRequest().query("FROM index-3,nonexisting-1"))) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index bf81befc69e81..1836daecbdfe7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -57,7 +57,8 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; public class IndexResolver { - private static Logger LOGGER = LogManager.getLogger(IndexResolver.class); + + private static final Logger LOGGER = LogManager.getLogger(IndexResolver.class); public static final Set ALL_FIELDS = Set.of("*"); public static final Set INDEX_METADATA_FIELD = Set.of(MetadataAttribute.INDEX); From 6fc3a7ed4c3d03377d369c43d68b891f023ec092 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 24 Nov 2025 10:04:21 +0100 Subject: [PATCH 7/7] use original indices --- .../xpack/esql/planner/PlannerUtils.java | 15 -------- .../xpack/esql/plugin/ComputeService.java | 3 +- .../xpack/esql/session/EsqlSession.java | 1 + .../xpack/esql/session/IndexResolver.java | 34 +++++++++++++++++-- .../xpack/esql/analysis/AnalyzerTests.java | 32 +++++++++++------ .../esql/type/EsqlDataTypeRegistryTests.java | 3 +- 6 files changed, 56 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index c457415783469..053cf8a4fc595 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.esql.planner; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.aggregation.AggregatorMode; @@ -61,14 +60,12 @@ import org.elasticsearch.xpack.esql.stats.SearchStats; import java.util.ArrayList; -import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; -import static java.util.Arrays.asList; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; @@ -155,18 +152,6 @@ private static ReducedPlan getPhysicalPlanReduction(int estimatedRowSize, Physic return new ReducedPlan(EstimatesRowSize.estimateRowSize(estimatedRowSize, plan)); } - /** - * Returns the original indices specified in the FROM command of the query. We need the original query to resolve alias filters. - */ - public static String[] planOriginalIndices(PhysicalPlan plan) { - if (plan == null) { - return Strings.EMPTY_ARRAY; - } - var indices = new LinkedHashSet(); - forEachRelation(plan, relation -> indices.addAll(asList(Strings.commaDelimitedListToStringArray(relation.indexPattern())))); - return indices.toArray(String[]::new); - } - public static boolean requiresSortedTimeSeriesSource(PhysicalPlan plan) { return plan.anyMatch(e -> { if (e instanceof FragmentExec f) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 9d270d830b2c3..163a0858c943c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -370,8 +370,7 @@ public void executePlan( return; } } - Map clusterToOriginalIndices = transportService.getRemoteClusterService() - .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, PlannerUtils.planOriginalIndices(physicalPlan)); + Map clusterToOriginalIndices = getIndices(physicalPlan, EsRelation::originalIndices); var localOriginalIndices = clusterToOriginalIndices.remove(LOCAL_CLUSTER); var localConcreteIndices = clusterToConcreteIndices.remove(LOCAL_CLUSTER); /* diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 8728010a75433..1d6aa2b53cc9e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -862,6 +862,7 @@ private void preAnalyzeMainIndices( indexMode == IndexMode.TIME_SERIES, preAnalysis.useAggregateMetricDoubleWhenNotSupported(), preAnalysis.useDenseVectorWhenNotSupported(), + indicesExpressionGrouper, listener.delegateFailureAndWrap((l, indexResolution) -> { EsqlCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.inner().failures()); l.onResponse( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 1836daecbdfe7..c187b7e3f2c7c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.indices.IndicesExpressionGrouper; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; @@ -49,6 +50,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; @@ -104,6 +106,7 @@ public void resolveIndices( includeAllDimensions, useAggregateMetricDoubleWhenNotSupported, useDenseVectorWhenNotSupported, + null, listener.map(Versioned::inner) ); } @@ -119,6 +122,7 @@ public void resolveIndicesVersioned( boolean includeAllDimensions, boolean useAggregateMetricDoubleWhenNotSupported, boolean useDenseVectorWhenNotSupported, + /* nullable */ IndicesExpressionGrouper indicesExpressionGrouper, ActionListener> listener ) { client.execute( @@ -133,7 +137,12 @@ public void resolveIndicesVersioned( useDenseVectorWhenNotSupported ); LOGGER.debug("minimum transport version {} {}", response.caps().minTransportVersion(), info.effectiveMinTransportVersion()); - l.onResponse(new Versioned<>(mergedMappings(indexWildcard, info), info.effectiveMinTransportVersion())); + l.onResponse( + new Versioned<>( + mergedMappings(indexWildcard, info, groupOriginalIndices(indicesExpressionGrouper)), + info.effectiveMinTransportVersion() + ) + ); }) ); } @@ -199,7 +208,11 @@ TransportVersion effectiveMinTransportVersion() { } // public for testing only - public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fieldsInfo) { + public static IndexResolution mergedMappings( + String indexPattern, + FieldsInfo fieldsInfo, + Function>> indexSplitter + ) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker int numberOfIndices = fieldsInfo.caps.getIndexResponses().size(); if (numberOfIndices == 0) { @@ -280,7 +293,10 @@ public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fie indexPattern, rootFields, allEmpty ? Map.of() : indexNameWithModes, - Map.of(), + // instead of using indexSplitter we could use original indices from + // FieldCapabilitiesResponse#resolvedLocally and FieldCapabilitiesResponse#resolvedRemotely + // once all remotes support it (v9.3+) + indexSplitter.apply(indexPattern), concreteIndices, partiallyUnmappedFields ); @@ -288,6 +304,18 @@ public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fie return IndexResolution.valid(index, indexNameWithModes.keySet(), failures); } + private static Function>> groupOriginalIndices(IndicesExpressionGrouper indicesExpressionGrouper) { + return indexPattern -> { + if (indicesExpressionGrouper == null) { + return Map.of(); + } + return Maps.transformValues( + indicesExpressionGrouper.groupIndices(IndicesOptions.DEFAULT, Strings.splitStringByCommaToArray(indexPattern), false), + v -> List.of(v.indices()) + ); + }; + } + private record IndexFieldCapabilitiesWithSourceHash(List fieldCapabilities, String indexMappingHash) {} private record CollectedFieldCaps( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index a45ec64af3ecd..152b3cd075ad4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3188,7 +3188,8 @@ public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordTyp ), List.of() ) - ) + ), + indexPattern -> Map.of() ); String query = "FROM foo, bar | INSIST_🐔 message"; @@ -3213,7 +3214,8 @@ public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeyword ), List.of() ) - ) + ), + indexPattern -> Map.of() ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); var limit = as(plan, Limit.class); @@ -3240,7 +3242,8 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyw ), List.of() ) - ) + ), + indexPattern -> Map.of() ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); var limit = as(plan, Limit.class); @@ -3265,7 +3268,8 @@ public void testResolveInsist_multiIndexSameMapping_fieldIsMapped() { ), List.of() ) - ) + ), + indexPattern -> Map.of() ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); var limit = as(plan, Limit.class); @@ -3290,7 +3294,8 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKe ), List.of() ) - ) + ), + indexPattern -> Map.of() ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); var limit = as(plan, Limit.class); @@ -3316,7 +3321,8 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa ), List.of() ) - ) + ), + indexPattern -> Map.of() ); VerificationException e = expectThrows( VerificationException.class, @@ -3338,7 +3344,8 @@ public void testResolveDenseVector() { { IndexResolution resolution = IndexResolver.mergedMappings( "foo", - new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, true) + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, true), + indexPattern -> Map.of() ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); @@ -3347,7 +3354,8 @@ public void testResolveDenseVector() { { IndexResolution resolution = IndexResolver.mergedMappings( "foo", - new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, false) + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, false), + indexPattern -> Map.of() ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); @@ -3369,7 +3377,8 @@ public void testResolveAggregateMetricDouble() { { IndexResolution resolution = IndexResolver.mergedMappings( "foo", - new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, true) + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, true), + indexPattern -> Map.of() ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); @@ -3381,7 +3390,8 @@ public void testResolveAggregateMetricDouble() { { IndexResolution resolution = IndexResolver.mergedMappings( "foo", - new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, false, true) + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, false, true), + indexPattern -> Map.of() ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); @@ -3833,7 +3843,7 @@ private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throw new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true, IndexMode.STANDARD) ); IndexResolver.FieldsInfo caps = fieldsInfoOnCurrentVersion(new FieldCapabilitiesResponse(idxResponses, List.of())); - IndexResolution resolution = IndexResolver.mergedMappings("test*", caps); + IndexResolution resolution = IndexResolver.mergedMappings("test*", caps, indexPattern -> Map.of()); var analyzer = analyzer(indexResolutions(resolution), TEST_VERIFIER, configuration(query)); return analyze(query, analyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index eda583457243c..182652027e0d7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -56,7 +56,8 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, // IndexResolver uses EsqlDataTypeRegistry directly IndexResolution resolution = IndexResolver.mergedMappings( "idx-*", - new IndexResolver.FieldsInfo(caps, TransportVersion.current(), false, false, false) + new IndexResolver.FieldsInfo(caps, TransportVersion.current(), false, false, false), + indexPattern -> Map.of() ); EsField f = resolution.get().mapping().get(field); assertThat(f.getDataType(), equalTo(expected));