From 1a848dfbd84121689ca11c5a81615075479d9072 Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Wed, 22 Jan 2025 16:08:13 +0000 Subject: [PATCH 1/5] Add test for system index migration using reindexing script --- modules/reindex/build.gradle | 3 + .../AbstractFeatureMigrationIntegTest.java | 13 +- .../migration/FeatureMigrationIT.java | 145 ++++++++++++++---- .../plugin-metadata/entitlement-policy.yaml | 2 + .../plugin-metadata/plugin-security.policy | 3 + ...ransportGetFeatureUpgradeStatusAction.java | 4 - 6 files changed, 128 insertions(+), 42 deletions(-) diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 47a3f51115b1d..05cd906f61160 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -44,6 +44,9 @@ dependencies { clusterModules project(':modules:lang-painless') clusterModules project(':modules:parent-join') clusterModules project(":modules:rest-root") + + internalClusterTestImplementation project(':modules:lang-painless') + internalClusterTestImplementation project(':modules:lang-painless:spi') } restResources { diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java index ea1c8ade00abe..099ec2897d18c 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java @@ -68,6 +68,7 @@ public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase static final String INTERNAL_MANAGED_INDEX_NAME = ".int-man-old"; static final int INDEX_DOC_COUNT = 100; // arbitrarily chosen static final int INTERNAL_MANAGED_FLAG_VALUE = 1; + static final String FIELD_NAME = "some_field"; public static final Version NEEDS_UPGRADE_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.previousMajor(); public static final IndexVersion NEEDS_UPGRADE_INDEX_VERSION = IndexVersionUtils.getPreviousMajorVersion( TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_INDEX_VERSION @@ -150,7 +151,7 @@ public T getPlugin(Class type) { return pluginsService.filterPlugins(type).findFirst().get(); } - public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) throws InterruptedException { + protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) { assertThat( "the strategy used below to create index names for descriptors without a primary index name only works for simple patterns", descriptor.getIndexPattern(), @@ -184,9 +185,13 @@ public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) thr CreateIndexResponse response = createRequest.get(); Assert.assertTrue(response.isShardsAcknowledged()); + indexDocs(indexName); + } + + protected void indexDocs(String indexName) { List docs = new ArrayList<>(INDEX_DOC_COUNT); for (int i = 0; i < INDEX_DOC_COUNT; i++) { - docs.add(ESIntegTestCase.prepareIndex(indexName).setId(Integer.toString(i)).setSource("some_field", "words words")); + docs.add(ESIntegTestCase.prepareIndex(indexName).setId(Integer.toString(i)).setSource(FIELD_NAME, "words words")); } indexRandom(true, docs); IndicesStatsResponse indexStats = ESIntegTestCase.indicesAdmin().prepareStats(indexName).setDocs(true).get(); @@ -211,7 +216,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna builder.field("dynamic", "strict"); builder.startObject("properties"); { - builder.startObject("some_field"); + builder.startObject(FIELD_NAME); builder.field("type", "keyword"); builder.endObject(); } @@ -225,7 +230,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna } } - public void assertIndexHasCorrectProperties( + protected void assertIndexHasCorrectProperties( Metadata metadata, String indexName, int settingsFlagValue, diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index a4aa0514bb47a..42c31976b4612 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; @@ -33,8 +34,11 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.upgrades.FeatureMigrationResults; import org.elasticsearch.upgrades.SingleFeatureMigrationResult; @@ -51,6 +55,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; @@ -62,6 +67,27 @@ import static org.hamcrest.Matchers.nullValue; public class FeatureMigrationIT extends AbstractFeatureMigrationIntegTest { + private static final String INTERNAL_MANAGED_WITH_SCRIPT_INDEX_NAME = ".int-mans-old"; + private static final String SCRIPTED_INDEX_FEATURE_NAME = "B-test-feature"; + private static final SystemIndexDescriptor INTERNAL_MANAGED_WITH_SCRIPT = SystemIndexDescriptor.builder() + .setIndexPattern(".int-mans-*") + .setAliasName(".internal-managed-with-script-alias") + .setPrimaryIndex(INTERNAL_MANAGED_WITH_SCRIPT_INDEX_NAME) + .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) + .setSettings(createSettings(NEEDS_UPGRADE_INDEX_VERSION, INTERNAL_MANAGED_FLAG_VALUE)) + .setMappings(createMapping(true, true)) + .setOrigin(ORIGIN) + .setVersionMetaKey(VERSION_META_KEY) + .setAllowedElasticProductOrigins(Collections.emptyList()) + .setMinimumNodeVersion(NEEDS_UPGRADE_VERSION) + .setPriorSystemIndexDescriptors(Collections.emptyList()) + .setMigrationScript(""" + if (ctx._source.some_field != null) { + ctx._source.some_field = 'migrated'; + } + """) + .build(); + @Override protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).build(); @@ -77,12 +103,14 @@ protected boolean forbidPrivateIndexSettings() { protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); plugins.add(TestPlugin.class); + plugins.add(SecondTestPlugin.class); plugins.add(ReindexPlugin.class); + plugins.add(PainlessPlugin.class); return plugins; } public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED); + createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); createSystemIndexForDescriptor(INTERNAL_UNMANAGED); createSystemIndexForDescriptor(EXTERNAL_MANAGED); createSystemIndexForDescriptor(EXTERNAL_UNMANAGED); @@ -102,7 +130,7 @@ public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { .stream() .map(PostFeatureUpgradeResponse.Feature::getFeatureName) .collect(Collectors.toSet()); - assertThat(migratingFeatures, hasItem(FEATURE_NAME)); + assertThat(migratingFeatures, hasItem(SCRIPTED_INDEX_FEATURE_NAME)); // We should see that the migration is in progress even though we just started the migration. assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS)); @@ -115,7 +143,7 @@ public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { }); } - public void testMigrateInternalManagedSystemIndex() throws Exception { + public void testMigrateSystemIndex() throws Exception { createSystemIndexForDescriptor(INTERNAL_MANAGED); createSystemIndexForDescriptor(INTERNAL_UNMANAGED); createSystemIndexForDescriptor(EXTERNAL_MANAGED); @@ -171,25 +199,7 @@ public void testMigrateInternalManagedSystemIndex() throws Exception { postUpgradeHookCalled.set(true); }); - PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT); - PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); - assertThat(migrationResponse.getReason(), nullValue()); - assertThat(migrationResponse.getElasticsearchException(), nullValue()); - final Set migratingFeatures = migrationResponse.getFeatures() - .stream() - .map(PostFeatureUpgradeResponse.Feature::getFeatureName) - .collect(Collectors.toSet()); - assertThat(migratingFeatures, hasItem(FEATURE_NAME)); - - GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT); - // The feature upgrade may take longer than ten seconds when tests are running - // in parallel, so we give assertBusy a sixty-second timeout. - assertBusy(() -> { - GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest) - .get(); - logger.info(Strings.toString(statusResponse)); - assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); - }, 60, TimeUnit.SECONDS); + executeMigration(FEATURE_NAME); // Waiting for shards to stabilize if indices were moved around ensureGreen(); @@ -197,14 +207,7 @@ public void testMigrateInternalManagedSystemIndex() throws Exception { assertTrue("the pre-migration hook wasn't actually called", preUpgradeHookCalled.get()); assertTrue("the post-migration hook wasn't actually called", postUpgradeHookCalled.get()); - Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata(); - // Check that the results metadata is what we expect. - FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE); - assertThat(currentResults, notNullValue()); - assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME))); - assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true)); - assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue()); - assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue()); + Metadata finalMetadata = assertMetadataAfterMigration(FEATURE_NAME); assertIndexHasCorrectProperties( finalMetadata, @@ -240,6 +243,18 @@ public void testMigrateInternalManagedSystemIndex() throws Exception { ); } + private static Metadata assertMetadataAfterMigration(String featureName) { + Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata(); + // Check that the results metadata is what we expect. + FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE); + assertThat(currentResults, notNullValue()); + assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(featureName))); + assertThat(currentResults.getFeatureStatuses().get(featureName).succeeded(), is(true)); + assertThat(currentResults.getFeatureStatuses().get(featureName).getFailedIndexName(), nullValue()); + assertThat(currentResults.getFeatureStatuses().get(featureName).getException(), nullValue()); + return finalMetadata; + } + public void testMigrateIndexWithWriteBlock() throws Exception { createSystemIndexForDescriptor(INTERNAL_UNMANAGED); @@ -260,7 +275,7 @@ public void testMigrateIndexWithWriteBlock() throws Exception { } public void testMigrationWillRunAfterError() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED); + createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); ensureGreen(); @@ -272,7 +287,7 @@ public void testMigrationWillRunAfterError() throws Exception { public ClusterState execute(ClusterState currentState) throws Exception { FeatureMigrationResults newResults = new FeatureMigrationResults( Collections.singletonMap( - FEATURE_NAME, + SCRIPTED_INDEX_FEATURE_NAME, SingleFeatureMigrationResult.failure(INTERNAL_MANAGED_INDEX_NAME, new RuntimeException("it failed :(")) ) ); @@ -304,8 +319,8 @@ public void onFailure(Exception e) { PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); // Make sure we actually started the migration assertTrue( - "could not find [" + FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), - migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(FEATURE_NAME)) + "could not find [" + SCRIPTED_INDEX_FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), + migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(SCRIPTED_INDEX_FEATURE_NAME)) ); // Now wait for the migration to finish (otherwise the test infra explodes) @@ -317,6 +332,51 @@ public void onFailure(Exception e) { }); } + private void executeMigration(String featureName) throws Exception { + PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT); + PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); + assertThat(migrationResponse.getReason(), nullValue()); + assertThat(migrationResponse.getElasticsearchException(), nullValue()); + final Set migratingFeatures = migrationResponse.getFeatures() + .stream() + .map(PostFeatureUpgradeResponse.Feature::getFeatureName) + .collect(Collectors.toSet()); + assertThat(migratingFeatures, hasItem(featureName)); + + GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT); + // The feature upgrade may take longer than ten seconds when tests are running + // in parallel, so we give assertBusy a sixty-second timeout. + assertBusy(() -> { + GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest) + .get(); + logger.info(Strings.toString(statusResponse)); + assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); + }, 60, TimeUnit.SECONDS); + } + + public void testMigrateUsingScript() throws Exception { + createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); + + executeMigration(SCRIPTED_INDEX_FEATURE_NAME); + ensureGreen(); + + Metadata metadata = assertMetadataAfterMigration(SCRIPTED_INDEX_FEATURE_NAME); + String newIndexName = ".int-mans-old-reindexed-for-9"; + assertIndexHasCorrectProperties( + metadata, + newIndexName, + INTERNAL_MANAGED_FLAG_VALUE, + true, + true, + Arrays.asList(".int-mans-old", ".internal-managed-with-script-alias") + ); + + SearchRequestBuilder searchRequestBuilder = prepareSearch(newIndexName) + .setQuery(QueryBuilders.termsQuery(FIELD_NAME, "migrated")) + .setSize(0); + assertHitCountAndNoFailures(searchRequestBuilder, INDEX_DOC_COUNT); + } + private String featureUpgradeErrorResponse(GetFeatureUpgradeStatusResponse statusResp) { return statusResp.getFeatureUpgradeStatuses() .stream() @@ -463,4 +523,21 @@ public void testMigrateWithTemplatesV2() throws Exception { assertThat(statusResp.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); }); } + + public static class SecondTestPlugin extends Plugin implements SystemIndexPlugin { + @Override + public String getFeatureName() { + return SCRIPTED_INDEX_FEATURE_NAME; + } + + @Override + public String getFeatureDescription() { + return "a plugin for testing system index migration"; + } + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(INTERNAL_MANAGED_WITH_SCRIPT); + } + } } diff --git a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml index df557f9944253..e9c8a53ef24be 100644 --- a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,4 @@ ALL-UNNAMED: - outbound_network +org.elasticsearch.painless: + - create_class_loader diff --git a/modules/reindex/src/main/plugin-metadata/plugin-security.policy b/modules/reindex/src/main/plugin-metadata/plugin-security.policy index 016cc6365b6ee..2b6d821c4d1a8 100644 --- a/modules/reindex/src/main/plugin-metadata/plugin-security.policy +++ b/modules/reindex/src/main/plugin-metadata/plugin-security.policy @@ -10,6 +10,9 @@ grant { // reindex opens socket connections using the rest client permission java.net.SocketPermission "*", "connect"; + + // needed for Painless to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; }; grant codeBase "${codebase.elasticsearch-rest-client}" { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java index 6f3e27981adf7..25ef2ae26d45e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java @@ -24,7 +24,6 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; -import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -59,7 +58,6 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_8_0_0; private final SystemIndices systemIndices; - PersistentTasksService persistentTasksService; @Inject public TransportGetFeatureUpgradeStatusAction( @@ -68,7 +66,6 @@ public TransportGetFeatureUpgradeStatusAction( ActionFilters actionFilters, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, - PersistentTasksService persistentTasksService, SystemIndices systemIndices ) { super( @@ -84,7 +81,6 @@ public TransportGetFeatureUpgradeStatusAction( ); this.systemIndices = systemIndices; - this.persistentTasksService = persistentTasksService; } @Override From b1fa2a717de74a854a586dcd1a7b4eb201b4e5ff Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Wed, 22 Jan 2025 16:18:06 +0000 Subject: [PATCH 2/5] Correct index/feature name --- .../elasticsearch/migration/FeatureMigrationIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index 42c31976b4612..8eacd5d4d57cf 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -110,7 +110,7 @@ protected Collection> nodePlugins() { } public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); + createSystemIndexForDescriptor(INTERNAL_MANAGED); createSystemIndexForDescriptor(INTERNAL_UNMANAGED); createSystemIndexForDescriptor(EXTERNAL_MANAGED); createSystemIndexForDescriptor(EXTERNAL_UNMANAGED); @@ -130,7 +130,7 @@ public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { .stream() .map(PostFeatureUpgradeResponse.Feature::getFeatureName) .collect(Collectors.toSet()); - assertThat(migratingFeatures, hasItem(SCRIPTED_INDEX_FEATURE_NAME)); + assertThat(migratingFeatures, hasItem(FEATURE_NAME)); // We should see that the migration is in progress even though we just started the migration. assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS)); @@ -275,7 +275,7 @@ public void testMigrateIndexWithWriteBlock() throws Exception { } public void testMigrationWillRunAfterError() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); + createSystemIndexForDescriptor(INTERNAL_MANAGED); ensureGreen(); @@ -287,7 +287,7 @@ public void testMigrationWillRunAfterError() throws Exception { public ClusterState execute(ClusterState currentState) throws Exception { FeatureMigrationResults newResults = new FeatureMigrationResults( Collections.singletonMap( - SCRIPTED_INDEX_FEATURE_NAME, + FEATURE_NAME, SingleFeatureMigrationResult.failure(INTERNAL_MANAGED_INDEX_NAME, new RuntimeException("it failed :(")) ) ); @@ -319,8 +319,8 @@ public void onFailure(Exception e) { PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); // Make sure we actually started the migration assertTrue( - "could not find [" + SCRIPTED_INDEX_FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), - migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(SCRIPTED_INDEX_FEATURE_NAME)) + "could not find [" + FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), + migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(FEATURE_NAME)) ); // Now wait for the migration to finish (otherwise the test infra explodes) From d534a3d13d3cfa51092ebbb952b0c3db23c505c7 Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Wed, 22 Jan 2025 18:58:41 +0000 Subject: [PATCH 3/5] Fix Existing System Index Migration Integration Tests (ES-10527) --- .../AbstractFeatureMigrationIntegTest.java | 9 ++------- .../migration/FeatureMigrationIT.java | 15 ++++++--------- .../migration/MultiFeatureMigrationIT.java | 10 +++++----- .../TransportGetFeatureUpgradeStatusAction.java | 8 +++++--- .../org/elasticsearch/indices/SystemIndices.java | 4 +++- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java index 099ec2897d18c..860d63000f124 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java @@ -69,10 +69,10 @@ public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase static final int INDEX_DOC_COUNT = 100; // arbitrarily chosen static final int INTERNAL_MANAGED_FLAG_VALUE = 1; static final String FIELD_NAME = "some_field"; - public static final Version NEEDS_UPGRADE_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.previousMajor(); - public static final IndexVersion NEEDS_UPGRADE_INDEX_VERSION = IndexVersionUtils.getPreviousMajorVersion( + protected static final IndexVersion NEEDS_UPGRADE_INDEX_VERSION = IndexVersionUtils.getPreviousMajorVersion( TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_INDEX_VERSION ); + protected static final int UPGRADED_TO_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.major + 1; static final SystemIndexDescriptor EXTERNAL_UNMANAGED = SystemIndexDescriptor.builder() .setIndexPattern(".ext-unman-*") @@ -132,11 +132,6 @@ public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase @Before public void setup() { - assumeTrue( - "We can only create the test indices we need if they're in the previous major version", - NEEDS_UPGRADE_VERSION.onOrAfter(Version.CURRENT.previousMajor()) - ); - internalCluster().setBootstrapMasterNodeIndex(0); masterName = internalCluster().startMasterOnlyNode(); masterAndDataNode = internalCluster().startNode(); diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index 8eacd5d4d57cf..cdf817a6b17b8 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -77,9 +77,7 @@ public class FeatureMigrationIT extends AbstractFeatureMigrationIntegTest { .setSettings(createSettings(NEEDS_UPGRADE_INDEX_VERSION, INTERNAL_MANAGED_FLAG_VALUE)) .setMappings(createMapping(true, true)) .setOrigin(ORIGIN) - .setVersionMetaKey(VERSION_META_KEY) .setAllowedElasticProductOrigins(Collections.emptyList()) - .setMinimumNodeVersion(NEEDS_UPGRADE_VERSION) .setPriorSystemIndexDescriptors(Collections.emptyList()) .setMigrationScript(""" if (ctx._source.some_field != null) { @@ -211,7 +209,7 @@ public void testMigrateSystemIndex() throws Exception { assertIndexHasCorrectProperties( finalMetadata, - ".int-man-old-reindexed-for-9", + ".int-man-old-reindexed-for-" + UPGRADED_TO_VERSION, INTERNAL_MANAGED_FLAG_VALUE, true, true, @@ -219,7 +217,7 @@ public void testMigrateSystemIndex() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".int-unman-old-reindexed-for-9", + ".int-unman-old-reindexed-for-" + UPGRADED_TO_VERSION, INTERNAL_UNMANAGED_FLAG_VALUE, false, true, @@ -227,7 +225,7 @@ public void testMigrateSystemIndex() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-man-old-reindexed-for-9", + ".ext-man-old-reindexed-for-" + UPGRADED_TO_VERSION, EXTERNAL_MANAGED_FLAG_VALUE, true, false, @@ -235,7 +233,7 @@ public void testMigrateSystemIndex() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-unman-old-reindexed-for-9", + ".ext-unman-old-reindexed-for-" + UPGRADED_TO_VERSION, EXTERNAL_UNMANAGED_FLAG_VALUE, false, false, @@ -361,7 +359,7 @@ public void testMigrateUsingScript() throws Exception { ensureGreen(); Metadata metadata = assertMetadataAfterMigration(SCRIPTED_INDEX_FEATURE_NAME); - String newIndexName = ".int-mans-old-reindexed-for-9"; + String newIndexName = ".int-mans-old-reindexed-for-" + UPGRADED_TO_VERSION; assertIndexHasCorrectProperties( metadata, newIndexName, @@ -371,8 +369,7 @@ public void testMigrateUsingScript() throws Exception { Arrays.asList(".int-mans-old", ".internal-managed-with-script-alias") ); - SearchRequestBuilder searchRequestBuilder = prepareSearch(newIndexName) - .setQuery(QueryBuilders.termsQuery(FIELD_NAME, "migrated")) + SearchRequestBuilder searchRequestBuilder = prepareSearch(newIndexName).setQuery(QueryBuilders.termsQuery(FIELD_NAME, "migrated")) .setSize(0); assertHitCountAndNoFailures(searchRequestBuilder, INDEX_DOC_COUNT); } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java index 3442e9dc43925..01a414243f39a 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java @@ -218,7 +218,7 @@ public void testMultipleFeatureMigration() throws Exception { // Finally, verify that all the indices exist and have the properties we expect. assertIndexHasCorrectProperties( finalMetadata, - ".int-man-old-reindexed-for-9", + ".int-man-old-reindexed-for-" + UPGRADED_TO_VERSION, INTERNAL_MANAGED_FLAG_VALUE, true, true, @@ -226,7 +226,7 @@ public void testMultipleFeatureMigration() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".int-unman-old-reindexed-for-9", + ".int-unman-old-reindexed-for-" + UPGRADED_TO_VERSION, INTERNAL_UNMANAGED_FLAG_VALUE, false, true, @@ -234,7 +234,7 @@ public void testMultipleFeatureMigration() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-man-old-reindexed-for-9", + ".ext-man-old-reindexed-for-" + UPGRADED_TO_VERSION, EXTERNAL_MANAGED_FLAG_VALUE, true, false, @@ -242,7 +242,7 @@ public void testMultipleFeatureMigration() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-unman-old-reindexed-for-9", + ".ext-unman-old-reindexed-for-" + UPGRADED_TO_VERSION, EXTERNAL_UNMANAGED_FLAG_VALUE, false, false, @@ -251,7 +251,7 @@ public void testMultipleFeatureMigration() throws Exception { assertIndexHasCorrectProperties( finalMetadata, - ".second-int-man-old-reindexed-for-9", + ".second-int-man-old-reindexed-for-" + UPGRADED_TO_VERSION, SECOND_FEATURE_IDX_FLAG_VALUE, true, true, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java index 25ef2ae26d45e..f7f023d9d478e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.indices.SystemIndices; @@ -52,10 +53,11 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA GetFeatureUpgradeStatusResponse> { /** - * Once all feature migrations for 9.x -> 10.x have been tested, we can bump this to Version.V_9_0_0 + * These versions should be set to current major and current major's index version */ - public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_8_0_0; - public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_8_0_0; + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) + public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_9_0_0; + public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.UPGRADE_TO_LUCENE_10_0_0; private final SystemIndices systemIndices; diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 42cda4da1a9e6..d01763d676f37 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -16,6 +16,7 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; @@ -110,7 +111,8 @@ public class SystemIndices { public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed"; public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin"; - public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-9"; + private static final int UPGRADED_TO_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.major + 1; + public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-" + UPGRADED_TO_VERSION; private static final Automaton EMPTY = Automata.makeEmpty(); From 4db8dc0ff65c2a0b19440dee1db7cc2b4d83dc76 Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Wed, 22 Jan 2025 19:28:49 +0000 Subject: [PATCH 4/5] Revert "Correct index/feature name" This reverts commit b1fa2a717de74a854a586dcd1a7b4eb201b4e5ff. --- .../elasticsearch/migration/FeatureMigrationIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index cdf817a6b17b8..ee75ce5f537f4 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -108,7 +108,7 @@ protected Collection> nodePlugins() { } public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED); + createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); createSystemIndexForDescriptor(INTERNAL_UNMANAGED); createSystemIndexForDescriptor(EXTERNAL_MANAGED); createSystemIndexForDescriptor(EXTERNAL_UNMANAGED); @@ -128,7 +128,7 @@ public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { .stream() .map(PostFeatureUpgradeResponse.Feature::getFeatureName) .collect(Collectors.toSet()); - assertThat(migratingFeatures, hasItem(FEATURE_NAME)); + assertThat(migratingFeatures, hasItem(SCRIPTED_INDEX_FEATURE_NAME)); // We should see that the migration is in progress even though we just started the migration. assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS)); @@ -273,7 +273,7 @@ public void testMigrateIndexWithWriteBlock() throws Exception { } public void testMigrationWillRunAfterError() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED); + createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); ensureGreen(); @@ -285,7 +285,7 @@ public void testMigrationWillRunAfterError() throws Exception { public ClusterState execute(ClusterState currentState) throws Exception { FeatureMigrationResults newResults = new FeatureMigrationResults( Collections.singletonMap( - FEATURE_NAME, + SCRIPTED_INDEX_FEATURE_NAME, SingleFeatureMigrationResult.failure(INTERNAL_MANAGED_INDEX_NAME, new RuntimeException("it failed :(")) ) ); @@ -317,8 +317,8 @@ public void onFailure(Exception e) { PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); // Make sure we actually started the migration assertTrue( - "could not find [" + FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), - migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(FEATURE_NAME)) + "could not find [" + SCRIPTED_INDEX_FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), + migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(SCRIPTED_INDEX_FEATURE_NAME)) ); // Now wait for the migration to finish (otherwise the test infra explodes) From 16cbdad70ea6aaf6bcb61551228d01253c90a8eb Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Wed, 22 Jan 2025 19:51:09 +0000 Subject: [PATCH 5/5] Revert "Add test for system index migration using reindexing script" This reverts commit 1a848dfb --- modules/reindex/build.gradle | 3 - .../AbstractFeatureMigrationIntegTest.java | 12 +- .../migration/FeatureMigrationIT.java | 142 +++++------------- .../plugin-metadata/entitlement-policy.yaml | 2 - .../plugin-metadata/plugin-security.policy | 3 - 5 files changed, 38 insertions(+), 124 deletions(-) diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 05cd906f61160..47a3f51115b1d 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -44,9 +44,6 @@ dependencies { clusterModules project(':modules:lang-painless') clusterModules project(':modules:parent-join') clusterModules project(":modules:rest-root") - - internalClusterTestImplementation project(':modules:lang-painless') - internalClusterTestImplementation project(':modules:lang-painless:spi') } restResources { diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java index 860d63000f124..2a1401242f81c 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java @@ -146,7 +146,7 @@ public T getPlugin(Class type) { return pluginsService.filterPlugins(type).findFirst().get(); } - protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) { + public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) throws InterruptedException { assertThat( "the strategy used below to create index names for descriptors without a primary index name only works for simple patterns", descriptor.getIndexPattern(), @@ -180,13 +180,9 @@ protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) CreateIndexResponse response = createRequest.get(); Assert.assertTrue(response.isShardsAcknowledged()); - indexDocs(indexName); - } - - protected void indexDocs(String indexName) { List docs = new ArrayList<>(INDEX_DOC_COUNT); for (int i = 0; i < INDEX_DOC_COUNT; i++) { - docs.add(ESIntegTestCase.prepareIndex(indexName).setId(Integer.toString(i)).setSource(FIELD_NAME, "words words")); + docs.add(ESIntegTestCase.prepareIndex(indexName).setId(Integer.toString(i)).setSource("some_field", "words words")); } indexRandom(true, docs); IndicesStatsResponse indexStats = ESIntegTestCase.indicesAdmin().prepareStats(indexName).setDocs(true).get(); @@ -211,7 +207,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna builder.field("dynamic", "strict"); builder.startObject("properties"); { - builder.startObject(FIELD_NAME); + builder.startObject("some_field"); builder.field("type", "keyword"); builder.endObject(); } @@ -225,7 +221,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna } } - protected void assertIndexHasCorrectProperties( + public void assertIndexHasCorrectProperties( Metadata metadata, String indexName, int settingsFlagValue, diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index ee75ce5f537f4..06233b6147824 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -21,7 +21,6 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; @@ -34,11 +33,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.SystemIndexDescriptor; -import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.upgrades.FeatureMigrationResults; import org.elasticsearch.upgrades.SingleFeatureMigrationResult; @@ -55,7 +51,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; @@ -67,25 +62,6 @@ import static org.hamcrest.Matchers.nullValue; public class FeatureMigrationIT extends AbstractFeatureMigrationIntegTest { - private static final String INTERNAL_MANAGED_WITH_SCRIPT_INDEX_NAME = ".int-mans-old"; - private static final String SCRIPTED_INDEX_FEATURE_NAME = "B-test-feature"; - private static final SystemIndexDescriptor INTERNAL_MANAGED_WITH_SCRIPT = SystemIndexDescriptor.builder() - .setIndexPattern(".int-mans-*") - .setAliasName(".internal-managed-with-script-alias") - .setPrimaryIndex(INTERNAL_MANAGED_WITH_SCRIPT_INDEX_NAME) - .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) - .setSettings(createSettings(NEEDS_UPGRADE_INDEX_VERSION, INTERNAL_MANAGED_FLAG_VALUE)) - .setMappings(createMapping(true, true)) - .setOrigin(ORIGIN) - .setAllowedElasticProductOrigins(Collections.emptyList()) - .setPriorSystemIndexDescriptors(Collections.emptyList()) - .setMigrationScript(""" - if (ctx._source.some_field != null) { - ctx._source.some_field = 'migrated'; - } - """) - .build(); - @Override protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).build(); @@ -101,14 +77,12 @@ protected boolean forbidPrivateIndexSettings() { protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); plugins.add(TestPlugin.class); - plugins.add(SecondTestPlugin.class); plugins.add(ReindexPlugin.class); - plugins.add(PainlessPlugin.class); return plugins; } public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); + createSystemIndexForDescriptor(INTERNAL_MANAGED); createSystemIndexForDescriptor(INTERNAL_UNMANAGED); createSystemIndexForDescriptor(EXTERNAL_MANAGED); createSystemIndexForDescriptor(EXTERNAL_UNMANAGED); @@ -128,7 +102,7 @@ public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { .stream() .map(PostFeatureUpgradeResponse.Feature::getFeatureName) .collect(Collectors.toSet()); - assertThat(migratingFeatures, hasItem(SCRIPTED_INDEX_FEATURE_NAME)); + assertThat(migratingFeatures, hasItem(FEATURE_NAME)); // We should see that the migration is in progress even though we just started the migration. assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS)); @@ -141,7 +115,7 @@ public void testStartMigrationAndImmediatelyCheckStatus() throws Exception { }); } - public void testMigrateSystemIndex() throws Exception { + public void testMigrateInternalManagedSystemIndex() throws Exception { createSystemIndexForDescriptor(INTERNAL_MANAGED); createSystemIndexForDescriptor(INTERNAL_UNMANAGED); createSystemIndexForDescriptor(EXTERNAL_MANAGED); @@ -197,7 +171,25 @@ public void testMigrateSystemIndex() throws Exception { postUpgradeHookCalled.set(true); }); - executeMigration(FEATURE_NAME); + PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT); + PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); + assertThat(migrationResponse.getReason(), nullValue()); + assertThat(migrationResponse.getElasticsearchException(), nullValue()); + final Set migratingFeatures = migrationResponse.getFeatures() + .stream() + .map(PostFeatureUpgradeResponse.Feature::getFeatureName) + .collect(Collectors.toSet()); + assertThat(migratingFeatures, hasItem(FEATURE_NAME)); + + GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT); + // The feature upgrade may take longer than ten seconds when tests are running + // in parallel, so we give assertBusy a sixty-second timeout. + assertBusy(() -> { + GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest) + .get(); + logger.info(Strings.toString(statusResponse)); + assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); + }, 60, TimeUnit.SECONDS); // Waiting for shards to stabilize if indices were moved around ensureGreen(); @@ -205,7 +197,14 @@ public void testMigrateSystemIndex() throws Exception { assertTrue("the pre-migration hook wasn't actually called", preUpgradeHookCalled.get()); assertTrue("the post-migration hook wasn't actually called", postUpgradeHookCalled.get()); - Metadata finalMetadata = assertMetadataAfterMigration(FEATURE_NAME); + Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata(); + // Check that the results metadata is what we expect. + FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE); + assertThat(currentResults, notNullValue()); + assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME))); + assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true)); + assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue()); + assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue()); assertIndexHasCorrectProperties( finalMetadata, @@ -241,18 +240,6 @@ public void testMigrateSystemIndex() throws Exception { ); } - private static Metadata assertMetadataAfterMigration(String featureName) { - Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata(); - // Check that the results metadata is what we expect. - FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE); - assertThat(currentResults, notNullValue()); - assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(featureName))); - assertThat(currentResults.getFeatureStatuses().get(featureName).succeeded(), is(true)); - assertThat(currentResults.getFeatureStatuses().get(featureName).getFailedIndexName(), nullValue()); - assertThat(currentResults.getFeatureStatuses().get(featureName).getException(), nullValue()); - return finalMetadata; - } - public void testMigrateIndexWithWriteBlock() throws Exception { createSystemIndexForDescriptor(INTERNAL_UNMANAGED); @@ -273,7 +260,7 @@ public void testMigrateIndexWithWriteBlock() throws Exception { } public void testMigrationWillRunAfterError() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); + createSystemIndexForDescriptor(INTERNAL_MANAGED); ensureGreen(); @@ -285,7 +272,7 @@ public void testMigrationWillRunAfterError() throws Exception { public ClusterState execute(ClusterState currentState) throws Exception { FeatureMigrationResults newResults = new FeatureMigrationResults( Collections.singletonMap( - SCRIPTED_INDEX_FEATURE_NAME, + FEATURE_NAME, SingleFeatureMigrationResult.failure(INTERNAL_MANAGED_INDEX_NAME, new RuntimeException("it failed :(")) ) ); @@ -317,8 +304,8 @@ public void onFailure(Exception e) { PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); // Make sure we actually started the migration assertTrue( - "could not find [" + SCRIPTED_INDEX_FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), - migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(SCRIPTED_INDEX_FEATURE_NAME)) + "could not find [" + FEATURE_NAME + "] in response: " + Strings.toString(migrationResponse), + migrationResponse.getFeatures().stream().anyMatch(feature -> feature.getFeatureName().equals(FEATURE_NAME)) ); // Now wait for the migration to finish (otherwise the test infra explodes) @@ -330,50 +317,6 @@ public void onFailure(Exception e) { }); } - private void executeMigration(String featureName) throws Exception { - PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT); - PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); - assertThat(migrationResponse.getReason(), nullValue()); - assertThat(migrationResponse.getElasticsearchException(), nullValue()); - final Set migratingFeatures = migrationResponse.getFeatures() - .stream() - .map(PostFeatureUpgradeResponse.Feature::getFeatureName) - .collect(Collectors.toSet()); - assertThat(migratingFeatures, hasItem(featureName)); - - GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT); - // The feature upgrade may take longer than ten seconds when tests are running - // in parallel, so we give assertBusy a sixty-second timeout. - assertBusy(() -> { - GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest) - .get(); - logger.info(Strings.toString(statusResponse)); - assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); - }, 60, TimeUnit.SECONDS); - } - - public void testMigrateUsingScript() throws Exception { - createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); - - executeMigration(SCRIPTED_INDEX_FEATURE_NAME); - ensureGreen(); - - Metadata metadata = assertMetadataAfterMigration(SCRIPTED_INDEX_FEATURE_NAME); - String newIndexName = ".int-mans-old-reindexed-for-" + UPGRADED_TO_VERSION; - assertIndexHasCorrectProperties( - metadata, - newIndexName, - INTERNAL_MANAGED_FLAG_VALUE, - true, - true, - Arrays.asList(".int-mans-old", ".internal-managed-with-script-alias") - ); - - SearchRequestBuilder searchRequestBuilder = prepareSearch(newIndexName).setQuery(QueryBuilders.termsQuery(FIELD_NAME, "migrated")) - .setSize(0); - assertHitCountAndNoFailures(searchRequestBuilder, INDEX_DOC_COUNT); - } - private String featureUpgradeErrorResponse(GetFeatureUpgradeStatusResponse statusResp) { return statusResp.getFeatureUpgradeStatuses() .stream() @@ -520,21 +463,4 @@ public void testMigrateWithTemplatesV2() throws Exception { assertThat(statusResp.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); }); } - - public static class SecondTestPlugin extends Plugin implements SystemIndexPlugin { - @Override - public String getFeatureName() { - return SCRIPTED_INDEX_FEATURE_NAME; - } - - @Override - public String getFeatureDescription() { - return "a plugin for testing system index migration"; - } - - @Override - public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.singletonList(INTERNAL_MANAGED_WITH_SCRIPT); - } - } } diff --git a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml index e9c8a53ef24be..df557f9944253 100644 --- a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,4 +1,2 @@ ALL-UNNAMED: - outbound_network -org.elasticsearch.painless: - - create_class_loader diff --git a/modules/reindex/src/main/plugin-metadata/plugin-security.policy b/modules/reindex/src/main/plugin-metadata/plugin-security.policy index 2b6d821c4d1a8..016cc6365b6ee 100644 --- a/modules/reindex/src/main/plugin-metadata/plugin-security.policy +++ b/modules/reindex/src/main/plugin-metadata/plugin-security.policy @@ -10,9 +10,6 @@ grant { // reindex opens socket connections using the rest client permission java.net.SocketPermission "*", "connect"; - - // needed for Painless to generate runtime classes - permission java.lang.RuntimePermission "createClassLoader"; }; grant codeBase "${codebase.elasticsearch-rest-client}" {