Skip to content

Field caps transport changes to return for each original expression what it was resolved to #136632

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
+732 −146
Open

Field caps transport changes to return for each original expression what it was resolved to #136632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
bf6f28a
alexey-ivanov-es work
piergm Oct 14, 2025
a706260
iter
piergm Oct 14, 2025
bfe54e6
field caps: local resolution working and tested
piergm Oct 15, 2025
9c08780
CCS impl
piergm Oct 15, 2025
1d6c55d
using CPS SAF resolution if present
piergm Oct 15, 2025
0774582
some TODOs to check later on
piergm Oct 15, 2025
d3a3fb3
Merge branch 'elastic:main' into field-caps-transport-changes
piergm Oct 15, 2025
430b867
[CI] Update transport version definitions
Oct 15, 2025
8e2520f
Merge branch 'main' into field-caps-transport-changes
piergm Oct 16, 2025
644b36e
iter
piergm Oct 16, 2025
4613f3f
merged main
piergm Oct 22, 2025
82d9ac6
iter
piergm Oct 22, 2025
dfb78fe
iter
piergm Oct 23, 2025
1e886da
iter
piergm Oct 23, 2025
57b456b
merged main, resolved conflicts
piergm Oct 23, 2025
6025cf8
Merge branch 'main' into field-caps-transport-changes
piergm Oct 23, 2025
b8ee242
Update docs/changelog/136632.yaml
piergm Oct 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/changelog/136632.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 136632
summary: Field caps transport changes to return for each original expression what
it was resolved to
area: Search
type: enhancement
issues: []
138 changes: 138 additions & 0 deletions ...c/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,11 @@
package org.elasticsearch.search.fieldcaps;

import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ResolvedIndexExpression;
import org.elasticsearch.action.ResolvedIndexExpressions;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IllegalIndexShardStateException;
Expand All @@ -21,18 +24,21 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.aMapWithSize;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.emptyArray;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;

public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase {

Expand Down Expand Up @@ -266,4 +272,136 @@ public void testReturnAllLocal() {
}
}
}

public void testResolvedToMatchingEverywhere() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might be difficult (impossible?) in the IT test framework, but is there a way to test that cross-cluster chaining does not occur here? (We'd need to link a remoteB to our remote-cluster, but not the origin cluster).

String localIndex = "index-local";
String remoteIndex = "index-remote";
String remoteClusterAlias = "remote_cluster";
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex);
FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster)
.setFields("*")
.setIncludeResolvedTo(true)
.get();

assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndexWithCluster));

ResolvedIndexExpressions local = response.getResolvedLocally();
assertThat(local, notNullValue());
assertThat(local.expressions(), hasSize(1));
assertEquals(
local.expressions().get(0).localExpressions().localIndexResolutionResult(),
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
);

List<String> localIndicesList = local.getLocalIndicesList();
assertThat(localIndicesList, hasSize(1));
assertThat(localIndicesList, containsInAnyOrder(localIndex));

Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
assertThat(remote, notNullValue());
assertThat(remote, aMapWithSize(1));
assertThat(remote.keySet(), contains(remoteClusterAlias));

ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
assertThat(remoteIndicesList, hasSize(1));
assertEquals(
remoteResponse.expressions().get(0).localExpressions().localIndexResolutionResult(),
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
);
assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex));
}

public void testResolvedToMatchingLocallyOnly() {
String localIndex = "index-local";
String remoteIndex = "index-remote";
String remoteClusterAlias = "remote_cluster";
String nonExistentIndex = "non-existent-index";
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
String remoteIndexWithCluster = String.join(":", remoteClusterAlias, nonExistentIndex);
FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster)
.setFields("*")
.setIncludeResolvedTo(true)
.get();

assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex));

ResolvedIndexExpressions local = response.getResolvedLocally();
assertThat(local, notNullValue());
assertThat(local.expressions(), hasSize(1));
assertEquals(
local.expressions().get(0).localExpressions().localIndexResolutionResult(),
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
);

List<String> localIndicesList = local.getLocalIndicesList();
assertThat(localIndicesList, hasSize(1));
assertThat(localIndicesList, containsInAnyOrder(localIndex));

Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
assertThat(remote, notNullValue());
assertThat(remote, aMapWithSize(1));
assertThat(remote.keySet(), contains(remoteClusterAlias));

ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
assertThat(remoteIndicesList, hasSize(0));
List<ResolvedIndexExpression> remoteResolvedExpressions = remoteResponse.expressions();
assertEquals(1, remoteResolvedExpressions.size());
assertEquals(
remoteResolvedExpressions.get(0).localExpressions().localIndexResolutionResult(),
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE
);
assertEquals(0, remoteIndicesList.size());
}

public void testResolvedToMatchingRemotelyOnly() {
String localIndex = "index-local";
String remoteIndex = "index-remote";
String remoteClusterAlias = "remote_cluster";
String nonExistentIndex = "non-existent-index";
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex);
boolean ignoreUnavailable = true;
IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);

FieldCapabilitiesResponse response = client().prepareFieldCaps(nonExistentIndex, remoteIndexWithCluster)
.setFields("*")
.setIncludeResolvedTo(true)
.setIndicesOptions(options) // without ignore unavaliable would throw error
.get();

assertThat(response.getIndices(), arrayContainingInAnyOrder(remoteIndexWithCluster));

ResolvedIndexExpressions local = response.getResolvedLocally();
assertThat(local, notNullValue());
assertThat(local.expressions(), hasSize(1));
assertEquals(
local.expressions().get(0).localExpressions().localIndexResolutionResult(),
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE
);

List<String> localIndicesList = local.getLocalIndicesList();
assertThat(localIndicesList, hasSize(0));

Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
assertThat(remote, notNullValue());
assertThat(remote, aMapWithSize(1));
assertThat(remote.keySet(), contains(remoteClusterAlias));

ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
assertThat(remoteIndicesList, hasSize(1));
assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex));
List<ResolvedIndexExpression> remoteResolvedExpressions = remoteResponse.expressions();
assertEquals(1, remoteResolvedExpressions.size());
ResolvedIndexExpression remoteExpression = remoteResolvedExpressions.get(0);
assertEquals(
remoteExpression.localExpressions().localIndexResolutionResult(),
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
);
assertEquals(1, remoteExpression.localExpressions().indices().size());
assertEquals(remoteIndex, remoteResolvedExpressions.get(0).original());
}
}
140 changes: 140 additions & 0 deletions .../src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import org.apache.http.entity.StringEntity;
import org.apache.logging.log4j.Level;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ResolvedIndexExpression;
import org.elasticsearch.action.ResolvedIndexExpressions;
import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
Expand Down Expand Up @@ -102,6 +104,7 @@
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;

Expand Down Expand Up @@ -913,6 +916,143 @@ public void testIndexMode() throws Exception {
assertThat(actualIndexModes, equalTo(indexModes));
}

public void testResolvedExpressionWithIndexAlias() {
FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").setIncludeResolvedTo(true).get();
assertIndices(response, "new_index");

assertEquals(0, response.getResolvedRemotely().size());
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
assertEquals(1, resolvedLocally.expressions().size());
ResolvedIndexExpression expression = expressions.get(0);
assertEquals("current", expression.original());
assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index"));
}

public void testResolvedExpressionWithWildcard() {
FieldCapabilitiesResponse response = client().prepareFieldCaps("*index").setFields("*").setIncludeResolvedTo(true).get();
assertIndices(response, "new_index", "old_index");

assertEquals(0, response.getResolvedRemotely().size());
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
assertEquals(1, resolvedLocally.expressions().size());
ResolvedIndexExpression expression = expressions.get(0);
assertEquals("*index", expression.original());
assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index"));
}

public void testResolvedExpressionWithClosedIndices() throws IOException {
// in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite
assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error"));
ensureGreen("index1-error", "index2-error");

// Closed shards will result to index error because shards must be in readable state
closeShards(internalCluster(), "index1-error", "index2-error");

FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error")
.setFields("*")
.setIncludeResolvedTo(true)
.get();
Set<String> openIndices = Set.of("old_index", "new_index");
Set<String> closedIndices = Set.of("index1-error", "index2-error");
assertEquals(0, response.getResolvedRemotely().size());
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
assertEquals(4, resolvedLocally.expressions().size());
for (ResolvedIndexExpression expression : expressions) {
ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
if (openIndices.contains(expression.original())) {
assertThat(expression.localExpressions().indices(), containsInAnyOrder(expression.original()));
assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult());
} else if (closedIndices.contains(expression.original())) {
Set<String> concreteIndices = localExpressions.indices();
assertEquals(0, concreteIndices.size());
assertEquals(
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE,
localExpressions.localIndexResolutionResult()
);
}
}
}

public void testResolvedExpressionWithAllIndices() {
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIncludeResolvedTo(true).get();
assertIndices(response, "new_index", "old_index");
assertEquals(0, response.getResolvedRemotely().size());
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
assertEquals(1, resolvedLocally.expressions().size());
ResolvedIndexExpression expression = expressions.get(0);
assertEquals("_all", expression.original()); // not setting indices means _all
ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index"));
assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult());
}

public void testResolvedExpressionWithOnlyOneClosedIndexAndIgnoreUnavailable() {
boolean ignoreUnavailable = true;
IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index")
.setFields("*")
.setIndicesOptions(options)
.setIncludeResolvedTo(true)
.get();

assertIndices(response);
assertEquals(0, response.getResolvedRemotely().size());
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
assertEquals(1, expressions.size());
ResolvedIndexExpression expression = expressions.get(0);
assertEquals("old_index", expression.original());
assertEquals(1, resolvedLocally.expressions().size());
ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
Set<String> concreteIndices = localExpressions.indices();
assertEquals(0, concreteIndices.size());
assertEquals(
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE,
localExpressions.localIndexResolutionResult()
);
}

public void testResolvedExpressionWithIndexFilter() throws InterruptedException {
assertAcked(
prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"),
prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long")
);

List<IndexRequestBuilder> reqs = new ArrayList<>();
reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08"));
reqs.add(prepareIndex("index-1").setSource("timestamp", "2018-07-08"));
reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12"));
reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08"));
indexRandom(true, reqs);

FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*")
.setFields("*")
.setIndexFilter(QueryBuilders.rangeQuery("timestamp").gte("2019-11-01"))
.setIncludeResolvedTo(true)
.get();

assertIndices(response, "index-2");
assertEquals(0, response.getResolvedRemotely().size());
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
assertEquals(1, resolvedLocally.expressions().size());
ResolvedIndexExpression expression = expressions.get(0);
assertEquals("index-*", expression.original());
assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-1", "index-2"));
}

public void testNoneExpressionIndices() {
// The auth code injects the pattern ["*", "-*"] which effectively means a request that requests no indices
FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "-*").setFields("*").get();

assertThat(response.getIndices().length, is(0));
}

private void assertIndices(FieldCapabilitiesResponse response, String... indices) {
assertNotNull(response.getIndices());
Arrays.sort(indices);
Expand Down
8 changes: 8 additions & 0 deletions server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,14 @@ public void addExpressions(
);
}

/**
* Add a new resolved expression.
* @param expression the expression you want to add.
*/
public void addExpression(ResolvedIndexExpression expression) {
expressions.add(expression);
}

public void addRemoteExpressions(String original, Set<String> remoteExpressions) {
Objects.requireNonNull(original);
Objects.requireNonNull(remoteExpressions);
Expand Down
Loading