diff --git a/docs/changelog/137558.yaml b/docs/changelog/137558.yaml new file mode 100644 index 0000000000000..c280173ee25b4 --- /dev/null +++ b/docs/changelog/137558.yaml @@ -0,0 +1,5 @@ +pr: 137558 +summary: Improve security migration resilience by handling version conflicts +area: Security +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 2e464afa72b76..08e2ecb615547 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -180,6 +180,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT = def(9_031_0_00, Version.LUCENE_10_2_2); public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW = def(9_032_0_00, Version.LUCENE_10_2_2); public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2); + public static final IndexVersion SECURITY_MIGRATIONS_METADATA_FLATTENED_UPDATE = def(9_034_0_00, Version.LUCENE_10_2_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index 4e1e92d6a1ec0..2c6dee150b7c2 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -42,6 +42,9 @@ dependencies { internalClusterTestImplementation(testArtifact(project(xpackModule('core')))) api "com.unboundid:unboundid-ldapsdk:${versions.ldapsdk}" + internalClusterTestImplementation project(path: ':modules:lang-painless') + internalClusterTestImplementation project(path: ':modules:lang-painless:spi') + // the following are all SAML dependencies - might as well download the whole internet api "org.opensaml:opensaml-core:${versions.opensaml}" api "org.opensaml:opensaml-saml-api:${versions.opensaml}" diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java index 9f67f3b900060..ecb7984591ef0 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java @@ -241,8 +241,6 @@ public void testMigrationFallbackNamePreCondition() throws Exception { waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations resetMigration(); - // Wait for the first migration to finish - waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION - 1); // Make sure migration didn't run yet (blocked by the fallback name) assertMigrationLessThan(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); @@ -315,10 +313,7 @@ public void testNewIndexSkipMigration() { ensureGreen(); deleteSecurityIndex(); // hack to force a new security index to be created ensureGreen(); - CountDownLatch awaitMigrations = awaitMigrationVersionUpdates( - masterNode, - SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION - ); + CountDownLatch awaitMigrations = awaitMigrationVersionUpdates(masterNode, SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey()); // Create a native role mapping to create security index and trigger migration createNativeRoleMapping("everyone_kibana_alone"); // Make sure no migration ran (set to current version without applying prior migrations) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/MetadataFlattenedMigrationIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/MetadataFlattenedMigrationIntegTests.java new file mode 100644 index 0000000000000..61c188df5e638 --- /dev/null +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/MetadataFlattenedMigrationIntegTests.java @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.SecurityIntegTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_DATA_KEY; +import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY; +import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ROLE_TYPE; +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.elasticsearch.xpack.security.support.SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +public class MetadataFlattenedMigrationIntegTests extends SecurityIntegTestCase { + + private final AtomicLong versionCounter = new AtomicLong(1); + + @Before + public void resetVersion() { + versionCounter.set(1); + } + + public void testMigrationWithConcurrentUpdates() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + internalCluster().startNode(); + ensureGreen(); + + waitForMigrationCompletion(); + var roles = createRoles(); + final var nativeRoleStore = internalCluster().getInstance(NativeRolesStore.class); + + try (ExecutorService executor = Executors.newSingleThreadExecutor()) { + final AtomicBoolean runUpdateRolesBackground = new AtomicBoolean(true); + executor.submit(() -> { + while (runUpdateRolesBackground.get()) { + // Only update half the list so the other half can be verified as migrated + RoleDescriptor roleToUpdate = randomFrom(roles.subList(0, roles.size() / 2)); + + RoleDescriptor updatedRole = new RoleDescriptor( + roleToUpdate.getName(), + new String[] { "monitor" }, + null, + null, + null, + null, + Map.of("test", "value", "timestamp", System.currentTimeMillis(), "random", randomAlphaOfLength(10)), + null + ); + nativeRoleStore.putRole( + WriteRequest.RefreshPolicy.IMMEDIATE, + updatedRole, + ActionListener.wrap(resp -> {}, ESTestCase::fail) + ); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + + resetMigration(); + try { + waitForMigrationCompletion(); + } finally { + runUpdateRolesBackground.set(false); + executor.shutdown(); + } + } + assertAllRolesHaveMetadataFlattened(); + } + + private void resetMigration() { + client().execute( + UpdateIndexMigrationVersionAction.INSTANCE, + new UpdateIndexMigrationVersionAction.Request( + TimeValue.MAX_VALUE, + ROLE_METADATA_FLATTENED_MIGRATION_VERSION - 1, + INTERNAL_SECURITY_MAIN_INDEX_7 + ) + ).actionGet(); + } + + private List createRoles() throws IOException { + var roles = randomList( + 25, + 50, + () -> new RoleDescriptor( + randomAlphaOfLength(20), + null, + null, + null, + null, + null, + Map.of("test", "value", "timestamp", System.currentTimeMillis(), "random", randomAlphaOfLength(10)), + Map.of() + ) + ); + for (RoleDescriptor role : roles) { + indexRoleDirectly(role); + } + indicesAdmin().prepareRefresh(INTERNAL_SECURITY_MAIN_INDEX_7).get(); + return roles; + } + + private void indexRoleDirectly(RoleDescriptor role) throws IOException { + XContentBuilder builder = buildRoleDocument(role); + prepareIndex(INTERNAL_SECURITY_MAIN_INDEX_7).setId(ROLE_TYPE + "-" + role.getName()) + .setSource(builder) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + } + + private XContentBuilder buildRoleDocument(RoleDescriptor role) throws IOException { + XContentBuilder builder = jsonBuilder().startObject(); + // metadata_flattened is populated by the native role store, so write directly to index to simulate pre-migration state + role.innerToXContent(builder, ToXContent.EMPTY_PARAMS, true); + builder.endObject(); + return builder; + } + + private int getCurrentMigrationVersion() { + ClusterService clusterService = internalCluster().getInstance(ClusterService.class); + IndexMetadata indexMetadata = clusterService.state().metadata().getProject().index(INTERNAL_SECURITY_MAIN_INDEX_7); + if (indexMetadata == null || indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) == null) { + return 0; + } + return Integer.parseInt(indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY).get(MIGRATION_VERSION_CUSTOM_DATA_KEY)); + } + + private void waitForMigrationCompletion() throws Exception { + assertBusy(() -> assertThat(getCurrentMigrationVersion(), greaterThanOrEqualTo(ROLE_METADATA_FLATTENED_MIGRATION_VERSION))); + } + + private void assertAllRolesHaveMetadataFlattened() { + SearchRequest searchRequest = new SearchRequest(INTERNAL_SECURITY_MAIN_INDEX_7); + searchRequest.source().query(QueryBuilders.termQuery("type", "role")).size(1000); + SearchResponse response = client().search(searchRequest).actionGet(); + for (SearchHit hit : response.getHits().getHits()) { + @SuppressWarnings("unchecked") + Map metadata = (Map) hit.getSourceAsMap().get("metadata_flattened"); + // Only check non-reserved roles + if (metadata.get("_reserved") == null) { + assertEquals("value", metadata.get("test")); + } + } + response.decRef(); + } + + @Override + protected Collection> nodePlugins() { + return Stream.concat(super.nodePlugins().stream(), Stream.of(PainlessPlugin.class)).collect(Collectors.toList()); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index dad6fddef371d..aebd37ba1980d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -488,7 +488,7 @@ public class Security extends Plugin public static final String SECURITY_CRYPTO_THREAD_POOL_NAME = XPackField.SECURITY + "-crypto"; - private static final int MAX_SECURITY_MIGRATION_RETRY_COUNT = 10; + private static final int MAX_SECURITY_MIGRATION_RETRY_COUNT = 1000; // TODO: ip filtering does not actually track license usage yet public static final LicensedFeature.Momentary IP_FILTERING_FEATURE = LicensedFeature.momentaryLenient( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index db3ef6b6f2397..a188c94c899a7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -50,10 +50,12 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; +import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams; import org.elasticsearch.xpack.security.SecurityFeatures; import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction; @@ -158,6 +160,7 @@ private IndexState unavailableState(ProjectId projectId, ProjectStatus status) { false, false, null, + false, null, null, null, @@ -180,6 +183,7 @@ public class IndexState { public final boolean mappingUpToDate; public final boolean createdOnLatestVersion; public final RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus; + public final boolean securityMigrationRunning; public final Integer migrationsVersion; // Min mapping version supported by the descriptors in the cluster public final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion; @@ -201,6 +205,7 @@ public IndexState( boolean mappingUpToDate, boolean createdOnLatestVersion, RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus, + boolean securityMigrationRunning, Integer migrationsVersion, SystemIndexDescriptor.MappingsVersion minClusterMappingVersion, Integer indexMappingVersion, @@ -220,6 +225,7 @@ public IndexState( this.migrationsVersion = migrationsVersion; this.createdOnLatestVersion = createdOnLatestVersion; this.roleMappingsCleanupMigrationStatus = roleMappingsCleanupMigrationStatus; + this.securityMigrationRunning = securityMigrationRunning; this.minClusterMappingVersion = minClusterMappingVersion; this.indexMappingVersion = indexMappingVersion; this.concreteIndexName = concreteIndexName; @@ -247,6 +253,7 @@ public boolean equals(Object o) { && mappingUpToDate == other.mappingUpToDate && createdOnLatestVersion == other.createdOnLatestVersion && roleMappingsCleanupMigrationStatus == other.roleMappingsCleanupMigrationStatus + && securityMigrationRunning == other.securityMigrationRunning && Objects.equals(indexMappingVersion, other.indexMappingVersion) && Objects.equals(migrationsVersion, other.migrationsVersion) && Objects.equals(minClusterMappingVersion, other.minClusterMappingVersion) @@ -268,6 +275,7 @@ public int hashCode() { mappingUpToDate, createdOnLatestVersion, roleMappingsCleanupMigrationStatus, + securityMigrationRunning, migrationsVersion, minClusterMappingVersion, indexMappingVersion, @@ -370,6 +378,8 @@ public String toString() { + createdOnLatestVersion + ", roleMappingsCleanupMigrationStatus=" + roleMappingsCleanupMigrationStatus + + ", securityMigrationRunning=" + + securityMigrationRunning + ", migrationsVersion=" + migrationsVersion + ", minClusterMappingVersion=" @@ -820,6 +830,9 @@ private IndexState updateProjectState(ProjectState project) { project, migrationsVersion ); + var persistentTaskCustomMetadata = PersistentTasksCustomMetadata.get(project.metadata()); + final boolean securityMigrationRunning = persistentTaskCustomMetadata != null + && persistentTaskCustomMetadata.getTask(SecurityMigrationTaskParams.TASK_NAME) != null; final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(project); final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion = getMinSecurityIndexMappingVersion(project); final int indexMappingVersion = loadIndexMappingVersion(systemIndexDescriptor.getAliasName(), project.metadata()); @@ -852,6 +865,7 @@ private IndexState updateProjectState(ProjectState project) { mappingIsUpToDate, createdOnLatestVersion, roleMappingsCleanupMigrationStatus, + securityMigrationRunning, migrationsVersion, minClusterMappingVersion, indexMappingVersion, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutor.java index 16fdbe3093728..fe2c4202d06ba 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutor.java @@ -54,7 +54,7 @@ public SecurityMigrationExecutor( @Override protected void nodeOperation(AllocatedPersistentTask task, SecurityMigrationTaskParams params, PersistentTaskState state) { ActionListener listener = ActionListener.wrap((res) -> task.markAsCompleted(), (exception) -> { - logger.warn("Security migration failed: " + exception); + logger.warn("Security migration failed", exception); task.markAsFailed(exception); }); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java index 2830a755cde66..f815a19c2b630 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.GroupedActionListener; @@ -89,35 +90,41 @@ default boolean checkPreConditions(SecurityIndexManager.IndexState securityIndex int minMappingVersion(); } - public static final Integer ROLE_METADATA_FLATTENED_MIGRATION_VERSION = 1; public static final Integer CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION = 2; + public static final Integer ROLE_METADATA_FLATTENED_MIGRATION_VERSION = 3; private static final Logger logger = LogManager.getLogger(SecurityMigration.class); public static final TreeMap MIGRATIONS_BY_VERSION = new TreeMap<>( Map.of( - ROLE_METADATA_FLATTENED_MIGRATION_VERSION, - new RoleMetadataFlattenedMigration(), CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION, - new CleanupRoleMappingDuplicatesMigration() + new CleanupRoleMappingDuplicatesMigration(), + ROLE_METADATA_FLATTENED_MIGRATION_VERSION, + new RoleMetadataFlattenedMigration() ) ); public static class RoleMetadataFlattenedMigration implements SecurityMigration { + @Override public void migrate(SecurityIndexManager indexManager, Client client, ActionListener listener) { BoolQueryBuilder filterQuery = new BoolQueryBuilder().filter(QueryBuilders.termQuery("type", "role")) .mustNot(QueryBuilders.existsQuery("metadata_flattened")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filterQuery).size(0).trackTotalHits(true); SearchRequest countRequest = new SearchRequest(indexManager.forCurrentProject().getConcreteIndexName()); countRequest.source(searchSourceBuilder); client.search(countRequest, ActionListener.wrap(response -> { - // If there are no roles, skip migration - if (response.getHits().getTotalHits().value() > 0) { - logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value() + "] roles"); - updateRolesByQuery(indexManager, client, filterQuery, listener); + if (response.isTimedOut() == false && response.getFailedShards() == 0) { + // If there are no roles, skip migration + if (response.getHits().getTotalHits() != null && response.getHits().getTotalHits().value() > 0) { + logger.info("Preparing to migrate [{}] roles", response.getHits().getTotalHits().value()); + updateRolesByQuery(indexManager, client, filterQuery, listener); + } else { + listener.onResponse(null); + } } else { - listener.onResponse(null); + listener.onFailure(new ElasticsearchException("metadata_flattened migration SearchRequest failed")); } }, listener::onFailure)); } @@ -129,15 +136,60 @@ private void updateRolesByQuery( ActionListener listener ) { UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexManager.forCurrentProject().getConcreteIndexName()); + updateByQueryRequest.setQuery(filterQuery); - updateByQueryRequest.setScript( - new Script(ScriptType.INLINE, "painless", "ctx._source.metadata_flattened = ctx._source.metadata", Collections.emptyMap()) - ); + updateByQueryRequest.setAbortOnVersionConflict(false); + updateByQueryRequest.setScript(new Script(ScriptType.INLINE, "painless", """ + if (ctx._source.metadata != null && ctx._source.metadata instanceof Map && !ctx._source.metadata.isEmpty()) { + ctx._source.metadata_flattened = ctx._source.metadata; + } else { + ctx.op = 'noop'; + } + """, Collections.emptyMap())); + client.admin() .cluster() .execute(UpdateByQueryAction.INSTANCE, updateByQueryRequest, ActionListener.wrap(bulkByScrollResponse -> { - logger.info("Migrated [" + bulkByScrollResponse.getTotal() + "] roles"); - listener.onResponse(null); + logger.debug( + "metadata_flattened update-by-query completed: total=[{}], updated=[{}], conflicts=[{}], failures=[{}], " + + "searchFailures=[{}], noops=[{}], timedOut=[{}]", + bulkByScrollResponse.getTotal(), + bulkByScrollResponse.getUpdated(), + bulkByScrollResponse.getVersionConflicts(), + bulkByScrollResponse.getBulkFailures().size(), + bulkByScrollResponse.getSearchFailures().size(), + bulkByScrollResponse.getNoops(), + bulkByScrollResponse.isTimedOut() + ); + if (bulkByScrollResponse.getBulkFailures().isEmpty() == false) { + listener.onFailure( + new ElasticsearchException( + "metadata_flattened migration update-by-query failed with bulk update failures [{}]", + bulkByScrollResponse.getBulkFailures() + ) + ); + } else if (bulkByScrollResponse.getSearchFailures().isEmpty() == false) { + listener.onFailure( + new ElasticsearchException( + "metadata_flattened migration update-by-query failed with search failures [{}]", + bulkByScrollResponse.getSearchFailures() + ) + ); + } else if (bulkByScrollResponse.isTimedOut()) { + listener.onFailure( + new ElasticsearchException( + "metadata_flattened migration update-by-query failed with timeout after [{}] seconds", + bulkByScrollResponse.getTook().seconds() + ) + ); + } else if (bulkByScrollResponse.getVersionConflicts() > 0) { + listener.onFailure( + new ElasticsearchException("metadata_flattened migration update-by-query failed with version conflicts") + ); + } else { + logger.info("metadata_flattened migration updated [{}] roles", bulkByScrollResponse.getUpdated()); + listener.onResponse(null); + } }, listener::onFailure)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 581da4e4dd016..b240d42fa3403 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2572,7 +2572,7 @@ private SecurityIndexManager.IndexState dummyState(ClusterHealthStatus indexStat return this.securityIndex.new IndexState( Metadata.DEFAULT_PROJECT_ID, SecurityIndexManager.ProjectStatus.PROJECT_AVAILABLE, Instant.now(), true, true, true, true, true, - null, null, null, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() + null, false, null, null, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 3da2648965e25..cb53f04b64715 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -36,7 +36,7 @@ public class NativeRealmTests extends ESTestCase { private SecurityIndexManager.IndexState dummyState(ClusterHealthStatus indexStatus) { return mock(SecurityIndexManager.class).new IndexState( Metadata.DEFAULT_PROJECT_ID, SecurityIndexManager.ProjectStatus.PROJECT_AVAILABLE, Instant.now(), true, true, true, true, true, - null, null, null, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() + null, false, null, null, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index d9c19b570e8e4..93b58407baff4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -400,7 +400,7 @@ private SecurityIndexManager.IndexState dummyState(ClusterHealthStatus indexStat private SecurityIndexManager.IndexState indexState(boolean isUpToDate, ClusterHealthStatus healthStatus) { return this.securityIndex.new IndexState( Metadata.DEFAULT_PROJECT_ID, SecurityIndexManager.ProjectStatus.PROJECT_AVAILABLE, Instant.now(), isUpToDate, true, true, true, - true, null, null, null, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() + true, null, false, null, null, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 3279988567ddf..d8ba9590ed5ed 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -2215,7 +2215,8 @@ public SecurityIndexManager.IndexState dummyIndexState(boolean isIndexUpToDate, var mgr = mock(SecurityIndexManager.class); return mgr.new IndexState( Metadata.DEFAULT_PROJECT_ID, SecurityIndexManager.ProjectStatus.PROJECT_AVAILABLE, Instant.now(), isIndexUpToDate, true, true, - true, true, null, null, null, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() + true, true, null, false, null, null, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid", Set + .of() ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 0e31fc4b363a2..14ceadfa97611 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -900,7 +900,8 @@ private SecurityIndexManager.IndexState dummyState( return securityIndex.new IndexState( Metadata.DEFAULT_PROJECT_ID, SecurityIndexManager.ProjectStatus.PROJECT_AVAILABLE, Instant.now(), isIndexUpToDate, true, true, - true, true, null, null, null, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid", Set.of() + true, true, null, false, null, null, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid", Set + .of() ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index 77c134639e095..5d882cd215083 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -59,11 +59,11 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( final ProjectId projectId = randomProjectIdOrDefault(); final SecurityIndexManager indexManager = mock(SecurityIndexManager.class); final SecurityIndexManager.IndexState previousState = indexManager.new IndexState( - projectId, SecurityIndexManager.ProjectStatus.CLUSTER_NOT_RECOVERED, null, false, false, false, false, false, null, null, null, - null, null, null, null, null, Set.of() + projectId, SecurityIndexManager.ProjectStatus.CLUSTER_NOT_RECOVERED, null, false, false, false, false, false, null, false, null, + null, null, null, null, null, null, Set.of() ); final SecurityIndexManager.IndexState currentState = indexManager.new IndexState( - projectId, SecurityIndexManager.ProjectStatus.PROJECT_AVAILABLE, Instant.now(), true, true, true, true, true, null, null, + projectId, SecurityIndexManager.ProjectStatus.PROJECT_AVAILABLE, Instant.now(), true, true, true, true, true, null, false, null, new SystemIndexDescriptor.MappingsVersion(SecurityMainIndexMappingVersion.latest().id(), 0), null, ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, "my_uuid", Set.of() ); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java index df8290327ee5a..afec1812baaa7 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java @@ -49,7 +49,7 @@ public void testRoleMigration() throws Exception { } } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { createRoleWithMetadata(upgradedTestRole, Map.of("meta", "test")); - waitForSecurityMigrationCompletion(adminClient(), 1); + waitForSecurityMigrationCompletion(adminClient(), 3); assertMigratedDocInSecurityIndex(oldTestRole, "meta", "test"); assertMigratedDocInSecurityIndex(mixed1TestRole, "meta", "test"); assertMigratedDocInSecurityIndex(mixed2TestRole, "meta", "test");