From 4b6a57e7c40df2c90731c24d9faea47a6f36c2e8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 27 Apr 2017 17:13:37 -0400 Subject: [PATCH 1/6] Add UpgraderPlugin The UpgraderPlugin adds two additional extension points called during cluster upgrade and when old indices are introduced into the cluster state during initial recovery, restore from a snapshot or as a dangling index. One extension point allows plugin to update old templates and another extension points allows to update/check index metadata. This change should simplify upgrade process for plugins that are using special indices to store their data, and should prevent situations when, for example, an accidental restore of an obsolete index would break the plugin. --- .../metadata/MetaDataIndexUpgradeService.java | 10 +- .../gateway/GatewayMetaState.java | 48 +++++-- .../java/org/elasticsearch/node/Node.java | 10 +- .../plugins/MetaDataUpgrader.java | 17 ++- .../org/elasticsearch/plugins/Plugin.java | 16 +-- .../elasticsearch/plugins/UpgraderPlugin.java | 69 ++++++++++ .../MetaDataIndexUpgradeServiceTests.java | 57 ++++++++- .../gateway/GatewayMetaStateTests.java | 119 ++++++++++++++---- .../indices/cluster/ClusterStateChanges.java | 3 +- 9 files changed, 283 insertions(+), 66 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 614d12547fc4d..892bfdc786abd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -35,8 +35,10 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.plugins.UpgraderPlugin; import java.util.AbstractMap; +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -54,14 +56,16 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { private final NamedXContentRegistry xContentRegistry; private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; + private final Collection upgraderPlugins; @Inject public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, - IndexScopedSettings indexScopedSettings) { + IndexScopedSettings indexScopedSettings, Collection upgraderPlugins) { super(settings); this.xContentRegistry = xContentRegistry; this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; + this.upgraderPlugins = upgraderPlugins; } /** @@ -84,6 +88,10 @@ public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version m newMetaData = archiveBrokenIndexSettings(newMetaData); // only run the check with the upgraded settings!! checkMappingsCompatibility(newMetaData); + // apply plugin checks + for (UpgraderPlugin upgraderPlugin : upgraderPlugins) { + newMetaData = upgraderPlugin.getIndexMetaDataUpgrader().apply(newMetaData); + } return markAsUpgraded(newMetaData); } diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 153ca8b3a8784..29a3acebee51a 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -50,6 +51,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.UnaryOperator; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -219,8 +223,8 @@ private void ensureNoPre019State() throws Exception { final String name = stateFile.getFileName().toString(); if (name.startsWith("metadata-")) { throw new IllegalStateException("Detected pre 0.19 metadata file please upgrade to a version before " - + Version.CURRENT.minimumCompatibilityVersion() - + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath()); + + Version.CURRENT.minimumCompatibilityVersion() + + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath()); } } } @@ -247,23 +251,41 @@ static MetaData upgradeMetaData(MetaData metaData, changed |= indexMetaData != newMetaData; upgradedMetaData.put(newMetaData, false); } - // collect current customs - Map existingCustoms = new HashMap<>(); - for (ObjectObjectCursor customCursor : metaData.customs()) { - existingCustoms.put(customCursor.key, customCursor.value); - } // upgrade global custom meta data - Map upgradedCustoms = metaDataUpgrader.customMetaDataUpgraders.apply(existingCustoms); - if (upgradedCustoms.equals(existingCustoms) == false) { - existingCustoms.keySet().forEach(upgradedMetaData::removeCustom); - for (Map.Entry upgradedCustomEntry : upgradedCustoms.entrySet()) { - upgradedMetaData.putCustom(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue()); - } + if (applyPluginUpraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders, + upgradedMetaData::removeCustom,upgradedMetaData::putCustom)) { + changed = true; + } + // upgrade current templates + if (applyPluginUpraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders, + upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) { changed = true; } return changed ? upgradedMetaData.build() : metaData; } + private static boolean applyPluginUpraders(ImmutableOpenMap existingData, + UnaryOperator> upgrader, + Consumer removeData, + BiConsumer putData) { + // collect current data + Map existingMap = new HashMap<>(); + for (ObjectObjectCursor customCursor : existingData) { + existingMap.put(customCursor.key, customCursor.value); + } + // upgrade global custom meta data + Map upgradedCustoms = upgrader.apply(existingMap); + if (upgradedCustoms.equals(existingMap) == false) { + // remove all data first so a plugin can remove custom metadata or templates if needed + existingMap.keySet().forEach(removeData); + for (Map.Entry upgradedCustomEntry : upgradedCustoms.entrySet()) { + putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue()); + } + return true; + } + return false; + } + // shard state BWC private void ensureNoPre019ShardState(NodeEnvironment nodeEnv) throws Exception { for (Path dataLocation : nodeEnv.nodeDataPaths()) { diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 01ccf68dc36aa..648556ef31dc7 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -115,6 +115,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.plugins.UpgraderPlugin; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.rest.RestController; import org.elasticsearch.script.ScriptModule; @@ -402,14 +403,11 @@ protected Node(final Environment environment, Collection .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptModule.getScriptService(), xContentRegistry).stream()) .collect(Collectors.toList()); - Collection>> customMetaDataUpgraders = - pluginsService.filterPlugins(Plugin.class).stream() - .map(Plugin::getCustomMetaDataUpgrader) - .collect(Collectors.toList()); + Collection upgraderPlugins = pluginsService.filterPlugins(UpgraderPlugin.class); final RestController restController = actionModule.getRestController(); final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class), threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController); - final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders); + final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(upgraderPlugins); final Transport transport = networkModule.getTransportSupplier().get(); final TransportService transportService = newTransportService(settings, transport, threadPool, networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings()); @@ -464,7 +462,7 @@ protected Node(final Environment environment, Collection b.bind(NetworkService.class).toInstance(networkService); b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService())); b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, - xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings())); + xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), upgraderPlugins)); b.bind(ClusterInfoService.class).toInstance(clusterInfoService); b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery()); { diff --git a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java index aaeddbec11974..3d423bfb82d88 100644 --- a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java +++ b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java @@ -19,6 +19,7 @@ package org.elasticsearch.plugins; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import java.util.Collection; @@ -32,13 +33,23 @@ public class MetaDataUpgrader { public final UnaryOperator> customMetaDataUpgraders; - public MetaDataUpgrader(Collection>> customMetaDataUpgraders) { + public final UnaryOperator> indexTemplateMetaDataUpgraders; + + public MetaDataUpgrader(Collection upgraderPlugins) { this.customMetaDataUpgraders = customs -> { Map upgradedCustoms = new HashMap<>(customs); - for (UnaryOperator> customMetaDataUpgrader : customMetaDataUpgraders) { - upgradedCustoms = customMetaDataUpgrader.apply(upgradedCustoms); + for (UpgraderPlugin upgraderPlugin : upgraderPlugins) { + upgradedCustoms = upgraderPlugin.getCustomMetaDataUpgrader().apply(upgradedCustoms); } return upgradedCustoms; }; + + this.indexTemplateMetaDataUpgraders = templates -> { + Map upgradedTemplates = new HashMap<>(templates); + for (UpgraderPlugin upgraderPlugin : upgraderPlugins) { + upgradedTemplates = upgraderPlugin.getIndexTemplateMetaDataUpgrader().apply(upgradedTemplates); + } + return upgradedTemplates; + }; } } diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index 87c5ef9a8c694..1e4673618b353 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -23,7 +23,6 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterModule; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; @@ -51,8 +50,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.UnaryOperator; /** * An extension point allowing to plug in custom functionality. This class has a number of extension points that are available to all @@ -68,6 +65,7 @@ *
  • {@link RepositoryPlugin} *
  • {@link ScriptPlugin} *
  • {@link SearchPlugin} + *
  • {@link UpgraderPlugin} * *

    In addition to extension points this class also declares some {@code @Deprecated} {@code public final void onModule} methods. These * methods should cause any extensions of {@linkplain Plugin} that used the pre-5.x style extension syntax to fail to build and point the @@ -149,18 +147,6 @@ public void onIndexModule(IndexModule indexModule) {} */ public List getSettingsFilter() { return Collections.emptyList(); } - /** - * Provides a function to modify global custom meta data on startup. - *

    - * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required. - * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map. - * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom} - * is unsupported - */ - public UnaryOperator> getCustomMetaDataUpgrader() { - return UnaryOperator.identity(); - } - /** * Provides the list of this plugin's custom thread pools, empty if * none. diff --git a/core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java b/core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java new file mode 100644 index 0000000000000..92db72d0b2b37 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.MetaData; + +import java.util.Map; +import java.util.function.UnaryOperator; + +/** + * Extension points for upgrading cluster and index metadata + */ +public class UpgraderPlugin { + /** + * Provides a function to modify global custom meta data on startup. + *

    + * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required. + * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map. + * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom} + * is unsupported + */ + public UnaryOperator> getCustomMetaDataUpgrader() { + return UnaryOperator.identity(); + } + + /** + * Provides a function to modify index template meta data on startup. + *

    + * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required. + * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map. + * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData} + * cannot be upgraded + */ + public UnaryOperator> getIndexTemplateMetaDataUpgrader() { + return UnaryOperator.identity(); + } + + /** + * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time. + *

    + * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required. + * @return Never {@code null}. The same or upgraded {@code IndexMetaData}. + * @throws IllegalStateException if the node should not start because the index is unsupported + */ + public UnaryOperator getIndexMetaDataUpgrader() { + return UnaryOperator.identity(); + } + + +} diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java index 24e969d06d62d..0aa89df7a1afd 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java @@ -22,16 +22,19 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.plugins.UpgraderPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import java.util.Collections; +import java.util.function.UnaryOperator; public class MetaDataIndexUpgradeServiceTests extends ESTestCase { public void testArchiveBrokenIndexSettings() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), - new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + Collections.emptyList()); IndexMetaData src = newIndexMeta("foo", Settings.EMPTY); IndexMetaData indexMetaData = service.archiveBrokenIndexSettings(src); assertSame(indexMetaData, src); @@ -58,7 +61,8 @@ public void testArchiveBrokenIndexSettings() { public void testUpgrade() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), - new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + Collections.emptyList()); IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build()); assertFalse(service.isUpgraded(src)); src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion()); @@ -70,7 +74,8 @@ public void testUpgrade() { public void testIsUpgraded() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), - new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + Collections.emptyList()); IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build()); assertFalse(service.isUpgraded(src)); Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion()); @@ -82,7 +87,8 @@ public void testIsUpgraded() { public void testFailUpgrade() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), - new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + Collections.emptyList()); final IndexMetaData metaData = newIndexMeta("foo", Settings.builder() .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.V_5_0_0_beta1) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.fromString("2.4.0")) @@ -99,6 +105,49 @@ public void testFailUpgrade() { service.upgradeIndexMetaData(goodMeta, Version.V_5_0_0.minimumIndexCompatibilityVersion()); } + public void testPluginUpgrade() { + MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator getIndexMetaDataUpgrader() { + return indexMetaData -> IndexMetaData.builder(indexMetaData) + .settings( + Settings.builder() + .put(indexMetaData.getSettings()) + .put("index.refresh_interval", "10s") + ).build(); + } + } + )); + IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "200s").build()); + assertFalse(service.isUpgraded(src)); + src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion()); + assertTrue(service.isUpgraded(src)); + assertEquals("10s", src.getSettings().get("index.refresh_interval")); + assertSame(src, service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion())); // no double upgrade + } + + public void testPluginUpgradeFailure() { + MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator getIndexMetaDataUpgrader() { + return indexMetaData -> { + throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName()); + }; + } + } + )); + IndexMetaData src = newIndexMeta("foo", Settings.EMPTY); + String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(src, + Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage(); + assertEquals(message, "Cannot upgrade index foo"); + } + public static IndexMetaData newIndexMeta(String name, Settings indexSettings) { Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 7b5530bba1061..7dc5fb03d698d 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -34,6 +35,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.plugins.UpgraderPlugin; import org.elasticsearch.test.TestCustomMetaData; import java.util.Arrays; @@ -41,7 +43,9 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import java.util.Set; +import java.util.function.UnaryOperator; import static java.util.Collections.emptySet; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; @@ -255,10 +259,17 @@ public void testWriteClosedIndex() throws Exception { public void testAddCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( - Collections.singletonList(customs -> { - customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); - return customs; - }) + Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return customs -> { + customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); + return customs; + }; + } + } + ) ); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); @@ -270,10 +281,16 @@ public void testAddCustomMetaDataOnUpgrade() throws Exception { public void testRemoveCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( - Collections.singletonList(customs -> { - customs.remove(CustomMetaData1.TYPE); - return customs; - }) + Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return customs -> { + customs.remove(CustomMetaData1.TYPE); + return customs; + }; + } + }) ); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); @@ -284,10 +301,16 @@ public void testRemoveCustomMetaDataOnUpgrade() throws Exception { public void testUpdateCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( - Collections.singletonList(customs -> { - customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); - return customs; - }) + Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return customs -> { + customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); + return customs; + }; + } + }) ); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); @@ -297,6 +320,28 @@ public void testUpdateCustomMetaDataOnUpgrade() throws Exception { assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1")); } + + public void testUpdateTemplateMetaDataOnUpgrade() throws Exception { + MetaData metaData = randomMetaData(); + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( + Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator> getIndexTemplateMetaDataUpgrader() { + return templates -> { + templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build()); + return templates; + }; + } + }) + ); + + MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); + assertTrue(upgrade != metaData); + assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData)); + assertTrue(upgrade.templates().containsKey("added_test_template")); + } + public void testNoMetaDataUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList()); @@ -311,8 +356,13 @@ public void testNoMetaDataUpgrade() throws Exception { public void testCustomMetaDataValidation() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( - customs -> { - throw new IllegalStateException("custom meta data too old"); + new UpgraderPlugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return customs -> { + throw new IllegalStateException("custom meta data too old"); + }; + } } )); try { @@ -334,17 +384,29 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception { case 2: metaData = randomMetaData(); break; - default: throw new IllegalStateException("should never happen"); + default: + throw new IllegalStateException("should never happen"); } MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( Arrays.asList( - customs -> { - customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); - return customs; + new UpgraderPlugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return + customs -> { + customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); + return customs; + }; + } }, - customs -> { - customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2")); - return customs; + new UpgraderPlugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return customs -> { + customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2")); + return customs; + }; + } }) ); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); @@ -372,7 +434,18 @@ public void testIndexMetaDataUpgrade() throws Exception { public void testCustomMetaDataNoChange() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); - MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new)); + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return HashMap::new; + } + + @Override + public UnaryOperator> getIndexTemplateMetaDataUpgrader() { + return HashMap::new; + } + })); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade == metaData); assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData)); @@ -385,7 +458,7 @@ private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgrad private final boolean upgrade; MockMetaDataIndexUpgradeService(boolean upgrade) { - super(Settings.EMPTY, null, null, null); + super(Settings.EMPTY, null, null, null, null); this.upgrade = upgrade; } @Override diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index f3bd58fd38a1c..e7c3024cfdc6b 100644 --- a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -162,7 +162,8 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th TransportService transportService = new TransportService(settings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings); - MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null) { + MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null, + null) { // metaData upgrader should do nothing @Override public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) { From 2c49da66089a4e9441cf9a0a4454ea003fc594e6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 28 Apr 2017 15:22:01 -0400 Subject: [PATCH 2/6] Add testIndexTemplateValidation test --- .../gateway/GatewayMetaStateTests.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 7dc5fb03d698d..f5a60baa8c290 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -454,6 +454,23 @@ public UnaryOperator> getIndexTemplateMetaDat } } + public void testIndexTemplateValidation() throws Exception { + MetaData metaData = randomMetaData(); + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( + new UpgraderPlugin() { + @Override + public UnaryOperator> getIndexTemplateMetaDataUpgrader() { + return customs -> { + throw new IllegalStateException("template is incompatible"); + }; + } + } + )); + String message = expectThrows(IllegalStateException.class, + () -> GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader)).getMessage(); + assertThat(message, equalTo("template is incompatible")); + } + private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgradeService { private final boolean upgrade; From 69ccb851800acd337cbb44339a4f14d80a8b07b9 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sat, 29 Apr 2017 16:57:12 -0400 Subject: [PATCH 3/6] applyPluginUpraders - > applyPluginUpgraders --- .../org/elasticsearch/gateway/GatewayMetaState.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 29a3acebee51a..d798f8b7745b1 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -252,22 +252,22 @@ static MetaData upgradeMetaData(MetaData metaData, upgradedMetaData.put(newMetaData, false); } // upgrade global custom meta data - if (applyPluginUpraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders, + if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders, upgradedMetaData::removeCustom,upgradedMetaData::putCustom)) { changed = true; } // upgrade current templates - if (applyPluginUpraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders, + if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders, upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) { changed = true; } return changed ? upgradedMetaData.build() : metaData; } - private static boolean applyPluginUpraders(ImmutableOpenMap existingData, - UnaryOperator> upgrader, - Consumer removeData, - BiConsumer putData) { + private static boolean applyPluginUpgraders(ImmutableOpenMap existingData, + UnaryOperator> upgrader, + Consumer removeData, + BiConsumer putData) { // collect current data Map existingMap = new HashMap<>(); for (ObjectObjectCursor customCursor : existingData) { From 97eed78895c6146e963f06ada5e5d3a8c0d0eb6a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 1 May 2017 11:14:30 -0400 Subject: [PATCH 4/6] Merge UpgraderPlugin methods into Plugin interface --- .../metadata/MetaDataIndexUpgradeService.java | 19 ++- .../java/org/elasticsearch/node/Node.java | 9 +- .../plugins/MetaDataUpgrader.java | 6 +- .../org/elasticsearch/plugins/Plugin.java | 41 +++++- .../elasticsearch/plugins/UpgraderPlugin.java | 69 ---------- .../MetaDataIndexUpgradeServiceTests.java | 6 +- .../gateway/GatewayMetaStateTests.java | 129 ++++++++++++++---- 7 files changed, 164 insertions(+), 115 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 892bfdc786abd..91cad88555909 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -35,13 +35,14 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.plugins.UpgraderPlugin; +import org.elasticsearch.plugins.Plugin; import java.util.AbstractMap; import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Set; +import java.util.function.UnaryOperator; /** * This service is responsible for upgrading legacy index metadata to the current version @@ -56,16 +57,22 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { private final NamedXContentRegistry xContentRegistry; private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; - private final Collection upgraderPlugins; + private final UnaryOperator upgraders; @Inject public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, - IndexScopedSettings indexScopedSettings, Collection upgraderPlugins) { + IndexScopedSettings indexScopedSettings, Collection upgraderPlugins) { super(settings); this.xContentRegistry = xContentRegistry; this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; - this.upgraderPlugins = upgraderPlugins; + this.upgraders = indexMetaData -> { + IndexMetaData newIndexMetaData = indexMetaData; + for (Plugin upgraderPlugin : upgraderPlugins) { + newIndexMetaData = upgraderPlugin.getIndexMetaDataUpgrader().apply(newIndexMetaData); + } + return newIndexMetaData; + }; } /** @@ -89,9 +96,7 @@ public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version m // only run the check with the upgraded settings!! checkMappingsCompatibility(newMetaData); // apply plugin checks - for (UpgraderPlugin upgraderPlugin : upgraderPlugins) { - newMetaData = upgraderPlugin.getIndexMetaDataUpgrader().apply(newMetaData); - } + newMetaData = upgraders.apply(newMetaData); return markAsUpgraded(newMetaData); } diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 648556ef31dc7..9499707862ccc 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -115,7 +115,6 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.plugins.UpgraderPlugin; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.rest.RestController; import org.elasticsearch.script.ScriptModule; @@ -403,11 +402,10 @@ protected Node(final Environment environment, Collection .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptModule.getScriptService(), xContentRegistry).stream()) .collect(Collectors.toList()); - Collection upgraderPlugins = pluginsService.filterPlugins(UpgraderPlugin.class); final RestController restController = actionModule.getRestController(); final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class), threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController); - final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(upgraderPlugins); + final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(pluginsService.filterPlugins(Plugin.class)); final Transport transport = networkModule.getTransportSupplier().get(); final TransportService transportService = newTransportService(settings, transport, threadPool, networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings()); @@ -461,8 +459,9 @@ protected Node(final Environment environment, Collection b.bind(TransportService.class).toInstance(transportService); b.bind(NetworkService.class).toInstance(networkService); b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService())); - b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, - xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), upgraderPlugins)); + b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, xContentRegistry, + indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), + pluginsService.filterPlugins(Plugin.class))); b.bind(ClusterInfoService.class).toInstance(clusterInfoService); b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery()); { diff --git a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java index 3d423bfb82d88..777421be32546 100644 --- a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java +++ b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java @@ -35,10 +35,10 @@ public class MetaDataUpgrader { public final UnaryOperator> indexTemplateMetaDataUpgraders; - public MetaDataUpgrader(Collection upgraderPlugins) { + public MetaDataUpgrader(Collection upgraderPlugins) { this.customMetaDataUpgraders = customs -> { Map upgradedCustoms = new HashMap<>(customs); - for (UpgraderPlugin upgraderPlugin : upgraderPlugins) { + for (Plugin upgraderPlugin : upgraderPlugins) { upgradedCustoms = upgraderPlugin.getCustomMetaDataUpgrader().apply(upgradedCustoms); } return upgradedCustoms; @@ -46,7 +46,7 @@ public MetaDataUpgrader(Collection upgraderPlugins) { this.indexTemplateMetaDataUpgraders = templates -> { Map upgradedTemplates = new HashMap<>(templates); - for (UpgraderPlugin upgraderPlugin : upgraderPlugins) { + for (Plugin upgraderPlugin : upgraderPlugins) { upgradedTemplates = upgraderPlugin.getIndexTemplateMetaDataUpgrader().apply(upgradedTemplates); } return upgradedTemplates; diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index 1e4673618b353..65502a96c5c82 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -23,6 +23,9 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; @@ -50,6 +53,8 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.UnaryOperator; /** * An extension point allowing to plug in custom functionality. This class has a number of extension points that are available to all @@ -65,7 +70,6 @@ *

  • {@link RepositoryPlugin} *
  • {@link ScriptPlugin} *
  • {@link SearchPlugin} - *
  • {@link UpgraderPlugin} * *

    In addition to extension points this class also declares some {@code @Deprecated} {@code public final void onModule} methods. These * methods should cause any extensions of {@linkplain Plugin} that used the pre-5.x style extension syntax to fail to build and point the @@ -147,6 +151,41 @@ public void onIndexModule(IndexModule indexModule) {} */ public List getSettingsFilter() { return Collections.emptyList(); } + /** + * Provides a function to modify global custom meta data on startup. + *

    + * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required. + * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map. + * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom} + * is unsupported + */ + public UnaryOperator> getCustomMetaDataUpgrader() { + return UnaryOperator.identity(); + } + + /** + * Provides a function to modify index template meta data on startup. + *

    + * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required. + * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map. + * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData} + * cannot be upgraded + */ + public UnaryOperator> getIndexTemplateMetaDataUpgrader() { + return UnaryOperator.identity(); + } + + /** + * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time. + *

    + * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required. + * @return Never {@code null}. The same or upgraded {@code IndexMetaData}. + * @throws IllegalStateException if the node should not start because the index is unsupported + */ + public UnaryOperator getIndexMetaDataUpgrader() { + return UnaryOperator.identity(); + } + /** * Provides the list of this plugin's custom thread pools, empty if * none. diff --git a/core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java b/core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java deleted file mode 100644 index 92db72d0b2b37..0000000000000 --- a/core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugins; - -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MetaData; - -import java.util.Map; -import java.util.function.UnaryOperator; - -/** - * Extension points for upgrading cluster and index metadata - */ -public class UpgraderPlugin { - /** - * Provides a function to modify global custom meta data on startup. - *

    - * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required. - * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map. - * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom} - * is unsupported - */ - public UnaryOperator> getCustomMetaDataUpgrader() { - return UnaryOperator.identity(); - } - - /** - * Provides a function to modify index template meta data on startup. - *

    - * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required. - * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map. - * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData} - * cannot be upgraded - */ - public UnaryOperator> getIndexTemplateMetaDataUpgrader() { - return UnaryOperator.identity(); - } - - /** - * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time. - *

    - * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required. - * @return Never {@code null}. The same or upgraded {@code IndexMetaData}. - * @throws IllegalStateException if the node should not start because the index is unsupported - */ - public UnaryOperator getIndexMetaDataUpgrader() { - return UnaryOperator.identity(); - } - - -} diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java index 0aa89df7a1afd..ccdef62a3ce98 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.plugins.UpgraderPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -109,7 +109,7 @@ public void testPluginUpgrade() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator getIndexMetaDataUpgrader() { return indexMetaData -> IndexMetaData.builder(indexMetaData) @@ -133,7 +133,7 @@ public void testPluginUpgradeFailure() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator getIndexMetaDataUpgrader() { return indexMetaData -> { diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index f5a60baa8c290..03ebae38f807c 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -34,8 +35,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.plugins.MetaDataUpgrader; -import org.elasticsearch.cluster.ESAllocationTestCase; -import org.elasticsearch.plugins.UpgraderPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.TestCustomMetaData; import java.util.Arrays; @@ -260,7 +260,7 @@ public void testAddCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator> getCustomMetaDataUpgrader() { return customs -> { @@ -282,7 +282,7 @@ public void testRemoveCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator> getCustomMetaDataUpgrader() { return customs -> { @@ -302,7 +302,7 @@ public void testUpdateCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator> getCustomMetaDataUpgrader() { return customs -> { @@ -325,7 +325,7 @@ public void testUpdateTemplateMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator> getIndexTemplateMetaDataUpgrader() { return templates -> { @@ -356,7 +356,7 @@ public void testNoMetaDataUpgrade() throws Exception { public void testCustomMetaDataValidation() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator> getCustomMetaDataUpgrader() { return customs -> { @@ -376,10 +376,10 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception { final MetaData metaData; switch (randomIntBetween(0, 2)) { case 0: - metaData = randomMetaData(new CustomMetaData1("data1"), new CustomMetaData2("data2")); + metaData = randomMetaDataWithIndexTemplates("template1", "template2"); break; case 1: - metaData = randomMetaData(randomBoolean() ? new CustomMetaData1("data1") : new CustomMetaData2("data2")); + metaData = randomMetaDataWithIndexTemplates(randomBoolean() ? "template1" : "template2"); break; case 2: metaData = randomMetaData(); @@ -389,22 +389,25 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception { } MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( Arrays.asList( - new UpgraderPlugin() { + new Plugin() { @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return - customs -> { - customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); - return customs; - }; + public UnaryOperator> getIndexTemplateMetaDataUpgrader() { + return indexTemplateMetaDatas -> { + indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1").settings( + Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build()).build()); + return indexTemplateMetaDatas; + + }; } }, - new UpgraderPlugin() { + new Plugin() { @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return customs -> { - customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2")); - return customs; + public UnaryOperator> getIndexTemplateMetaDataUpgrader() { + return indexTemplateMetaDatas -> { + indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2").settings( + Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build()); + return indexTemplateMetaDatas; + }; } }) @@ -412,10 +415,10 @@ public UnaryOperator> getCustomMetaDataUpgrader() { MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData)); - assertNotNull(upgrade.custom(CustomMetaData1.TYPE)); - assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1")); - assertNotNull(upgrade.custom(CustomMetaData2.TYPE)); - assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData2.TYPE)).getData(), equalTo("modified_data2")); + assertNotNull(upgrade.templates().get("template1")); + assertThat(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(upgrade.templates().get("template1").settings()), equalTo(20)); + assertNotNull(upgrade.templates().get("template2")); + assertThat(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(upgrade.templates().get("template2").settings()), equalTo(10)); for (IndexMetaData indexMetaData : upgrade) { assertTrue(metaData.hasIndexMetaData(indexMetaData)); } @@ -435,7 +438,7 @@ public void testIndexMetaDataUpgrade() throws Exception { public void testCustomMetaDataNoChange() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator> getCustomMetaDataUpgrader() { return HashMap::new; @@ -457,7 +460,7 @@ public UnaryOperator> getIndexTemplateMetaDat public void testIndexTemplateValidation() throws Exception { MetaData metaData = randomMetaData(); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( - new UpgraderPlugin() { + new Plugin() { @Override public UnaryOperator> getIndexTemplateMetaDataUpgrader() { return customs -> { @@ -471,6 +474,56 @@ public UnaryOperator> getIndexTemplateMetaDat assertThat(message, equalTo("template is incompatible")); } + + public void testMultipleIndexTemplateUpgrade() throws Exception { + final MetaData metaData; + switch (randomIntBetween(0, 2)) { + case 0: + metaData = randomMetaData(new CustomMetaData1("data1"), new CustomMetaData2("data2")); + break; + case 1: + metaData = randomMetaData(randomBoolean() ? new CustomMetaData1("data1") : new CustomMetaData2("data2")); + break; + case 2: + metaData = randomMetaData(); + break; + default: + throw new IllegalStateException("should never happen"); + } + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( + Arrays.asList( + new Plugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return + customs -> { + customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); + return customs; + }; + } + }, + new Plugin() { + @Override + public UnaryOperator> getCustomMetaDataUpgrader() { + return customs -> { + customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2")); + return customs; + }; + } + }) + ); + MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); + assertTrue(upgrade != metaData); + assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData)); + assertNotNull(upgrade.custom(CustomMetaData1.TYPE)); + assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1")); + assertNotNull(upgrade.custom(CustomMetaData2.TYPE)); + assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData2.TYPE)).getData(), equalTo("modified_data2")); + for (IndexMetaData indexMetaData : upgrade) { + assertTrue(metaData.hasIndexMetaData(indexMetaData)); + } + } + private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgradeService { private final boolean upgrade; @@ -478,6 +531,7 @@ private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgrad super(Settings.EMPTY, null, null, null, null); this.upgrade = upgrade; } + @Override public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) { return upgrade ? IndexMetaData.builder(indexMetaData).build() : indexMetaData; @@ -535,4 +589,25 @@ private static MetaData randomMetaData(TestCustomMetaData... customMetaDatas) { } return builder.build(); } + + private static MetaData randomMetaDataWithIndexTemplates(String... templates) { + MetaData.Builder builder = MetaData.builder(); + for (String template : templates) { + IndexTemplateMetaData templateMetaData = IndexTemplateMetaData.builder(template) + .settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 3)) + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5))) + .build(); + builder.put(templateMetaData); + } + for (int i = 0; i < randomIntBetween(1, 5); i++) { + builder.put( + IndexMetaData.builder(randomAlphaOfLength(10)) + .settings(settings(Version.CURRENT)) + .numberOfReplicas(randomIntBetween(0, 3)) + .numberOfShards(randomIntBetween(1, 5)) + ); + } + return builder.build(); + } } From f53f7eb624e9e2295c3b40c7f6ec72b996096859 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 1 May 2017 14:52:40 -0400 Subject: [PATCH 5/6] Decouple upgraders from Plugins --- .../metadata/MetaDataIndexUpgradeService.java | 7 +- .../java/org/elasticsearch/node/Node.java | 17 +- .../plugins/MetaDataUpgrader.java | 11 +- .../MetaDataIndexUpgradeServiceTests.java | 29 +-- .../gateway/GatewayMetaStateTests.java | 196 ++++++------------ 5 files changed, 99 insertions(+), 161 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 91cad88555909..5280be3e78ac2 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -61,15 +61,16 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { @Inject public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, - IndexScopedSettings indexScopedSettings, Collection upgraderPlugins) { + IndexScopedSettings indexScopedSettings, + Collection> indexMetaDataUpgraders) { super(settings); this.xContentRegistry = xContentRegistry; this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; this.upgraders = indexMetaData -> { IndexMetaData newIndexMetaData = indexMetaData; - for (Plugin upgraderPlugin : upgraderPlugins) { - newIndexMetaData = upgraderPlugin.getIndexMetaDataUpgrader().apply(newIndexMetaData); + for (UnaryOperator upgrader : indexMetaDataUpgraders) { + newIndexMetaData = upgrader.apply(newIndexMetaData); } return newIndexMetaData; }; diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 9499707862ccc..75f3db1e8b086 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -43,6 +43,8 @@ import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.NodeConnectionsService; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -405,7 +407,17 @@ protected Node(final Environment environment, Collection final RestController restController = actionModule.getRestController(); final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class), threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController); - final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(pluginsService.filterPlugins(Plugin.class)); + Collection>> customMetaDataUpgraders = + pluginsService.filterPlugins(Plugin.class).stream() + .map(Plugin::getCustomMetaDataUpgrader) + .collect(Collectors.toList()); + Collection>> indexTemplateMetaDataUpgraders = + pluginsService.filterPlugins(Plugin.class).stream() + .map(Plugin::getIndexTemplateMetaDataUpgrader) + .collect(Collectors.toList()); + Collection> indexMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream() + .map(Plugin::getIndexMetaDataUpgrader).collect(Collectors.toList()); + final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders, indexTemplateMetaDataUpgraders); final Transport transport = networkModule.getTransportSupplier().get(); final TransportService transportService = newTransportService(settings, transport, threadPool, networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings()); @@ -460,8 +472,7 @@ protected Node(final Environment environment, Collection b.bind(NetworkService.class).toInstance(networkService); b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService())); b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, xContentRegistry, - indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), - pluginsService.filterPlugins(Plugin.class))); + indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders)); b.bind(ClusterInfoService.class).toInstance(clusterInfoService); b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery()); { diff --git a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java index 777421be32546..49dd91901d756 100644 --- a/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java +++ b/core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java @@ -35,19 +35,20 @@ public class MetaDataUpgrader { public final UnaryOperator> indexTemplateMetaDataUpgraders; - public MetaDataUpgrader(Collection upgraderPlugins) { + public MetaDataUpgrader(Collection>> customMetaDataUpgraders, + Collection>> indexTemplateMetaDataUpgraders) { this.customMetaDataUpgraders = customs -> { Map upgradedCustoms = new HashMap<>(customs); - for (Plugin upgraderPlugin : upgraderPlugins) { - upgradedCustoms = upgraderPlugin.getCustomMetaDataUpgrader().apply(upgradedCustoms); + for (UnaryOperator> customMetaDataUpgrader : customMetaDataUpgraders) { + upgradedCustoms = customMetaDataUpgrader.apply(upgradedCustoms); } return upgradedCustoms; }; this.indexTemplateMetaDataUpgraders = templates -> { Map upgradedTemplates = new HashMap<>(templates); - for (Plugin upgraderPlugin : upgraderPlugins) { - upgradedTemplates = upgraderPlugin.getIndexTemplateMetaDataUpgrader().apply(upgradedTemplates); + for (UnaryOperator> upgrader : indexTemplateMetaDataUpgraders) { + upgradedTemplates = upgrader.apply(upgradedTemplates); } return upgradedTemplates; }; diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java index ccdef62a3ce98..98dad8810d8cc 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java @@ -22,12 +22,10 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import java.util.Collections; -import java.util.function.UnaryOperator; public class MetaDataIndexUpgradeServiceTests extends ESTestCase { @@ -109,18 +107,12 @@ public void testPluginUpgrade() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator getIndexMetaDataUpgrader() { - return indexMetaData -> IndexMetaData.builder(indexMetaData) - .settings( - Settings.builder() - .put(indexMetaData.getSettings()) - .put("index.refresh_interval", "10s") - ).build(); - } - } - )); + indexMetaData -> IndexMetaData.builder(indexMetaData) + .settings( + Settings.builder() + .put(indexMetaData.getSettings()) + .put("index.refresh_interval", "10s") + ).build())); IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "200s").build()); assertFalse(service.isUpgraded(src)); src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion()); @@ -133,13 +125,8 @@ public void testPluginUpgradeFailure() { MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator getIndexMetaDataUpgrader() { - return indexMetaData -> { - throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName()); - }; - } + indexMetaData -> { + throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName()); } )); IndexMetaData src = newIndexMeta("foo", Settings.EMPTY); diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 03ebae38f807c..e7daa9a791d43 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.plugins.MetaDataUpgrader; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.TestCustomMetaData; import java.util.Arrays; @@ -43,9 +42,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; -import java.util.Map; import java.util.Set; -import java.util.function.UnaryOperator; import static java.util.Collections.emptySet; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; @@ -259,17 +256,11 @@ public void testWriteClosedIndex() throws Exception { public void testAddCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( - Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return customs -> { - customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); - return customs; - }; - } - } - ) + Collections.singletonList(customs -> { + customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); + return customs; + }), + Collections.emptyList() ); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); @@ -281,16 +272,11 @@ public UnaryOperator> getCustomMetaDataUpgrader() { public void testRemoveCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( - Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return customs -> { - customs.remove(CustomMetaData1.TYPE); - return customs; - }; - } - }) + Collections.singletonList(customs -> { + customs.remove(CustomMetaData1.TYPE); + return customs; + }), + Collections.emptyList() ); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); @@ -301,16 +287,11 @@ public UnaryOperator> getCustomMetaDataUpgrader() { public void testUpdateCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( - Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return customs -> { - customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); - return customs; - }; - } - }) + Collections.singletonList(customs -> { + customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); + return customs; + }), + Collections.emptyList() ); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); @@ -324,17 +305,13 @@ public UnaryOperator> getCustomMetaDataUpgrader() { public void testUpdateTemplateMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( + Collections.emptyList(), Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator> getIndexTemplateMetaDataUpgrader() { - return templates -> { - templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build()); - return templates; - }; - } - }) - ); + templates -> { + templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build()); + return templates; + } + )); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); @@ -344,7 +321,7 @@ public UnaryOperator> getIndexTemplateMetaDat public void testNoMetaDataUpgrade() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); - MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList()); + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList()); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade == metaData); assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData)); @@ -356,15 +333,10 @@ public void testNoMetaDataUpgrade() throws Exception { public void testCustomMetaDataValidation() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return customs -> { - throw new IllegalStateException("custom meta data too old"); - }; - } + customs -> { + throw new IllegalStateException("custom meta data too old"); } - )); + ), Collections.emptyList()); try { GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); } catch (IllegalStateException e) { @@ -376,10 +348,10 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception { final MetaData metaData; switch (randomIntBetween(0, 2)) { case 0: - metaData = randomMetaDataWithIndexTemplates("template1", "template2"); + metaData = randomMetaData(new CustomMetaData1("data1"), new CustomMetaData2("data2")); break; case 1: - metaData = randomMetaDataWithIndexTemplates(randomBoolean() ? "template1" : "template2"); + metaData = randomMetaData(randomBoolean() ? new CustomMetaData1("data1") : new CustomMetaData2("data2")); break; case 2: metaData = randomMetaData(); @@ -389,36 +361,22 @@ public void testMultipleCustomMetaDataUpgrade() throws Exception { } MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( Arrays.asList( - new Plugin() { - @Override - public UnaryOperator> getIndexTemplateMetaDataUpgrader() { - return indexTemplateMetaDatas -> { - indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1").settings( - Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build()).build()); - return indexTemplateMetaDatas; - - }; - } + customs -> { + customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); + return customs; }, - new Plugin() { - @Override - public UnaryOperator> getIndexTemplateMetaDataUpgrader() { - return indexTemplateMetaDatas -> { - indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2").settings( - Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build()); - return indexTemplateMetaDatas; - - }; - } - }) - ); + customs -> { + customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2")); + return customs; + } + ), Collections.emptyList()); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData)); - assertNotNull(upgrade.templates().get("template1")); - assertThat(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(upgrade.templates().get("template1").settings()), equalTo(20)); - assertNotNull(upgrade.templates().get("template2")); - assertThat(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(upgrade.templates().get("template2").settings()), equalTo(10)); + assertNotNull(upgrade.custom(CustomMetaData1.TYPE)); + assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1")); + assertNotNull(upgrade.custom(CustomMetaData2.TYPE)); + assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData2.TYPE)).getData(), equalTo("modified_data2")); for (IndexMetaData indexMetaData : upgrade) { assertTrue(metaData.hasIndexMetaData(indexMetaData)); } @@ -426,7 +384,7 @@ public UnaryOperator> getIndexTemplateMetaDat public void testIndexMetaDataUpgrade() throws Exception { MetaData metaData = randomMetaData(); - MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList()); + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList()); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(true), metaDataUpgrader); assertTrue(upgrade != metaData); assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData)); @@ -437,18 +395,8 @@ public void testIndexMetaDataUpgrade() throws Exception { public void testCustomMetaDataNoChange() throws Exception { MetaData metaData = randomMetaData(new CustomMetaData1("data")); - MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return HashMap::new; - } - - @Override - public UnaryOperator> getIndexTemplateMetaDataUpgrader() { - return HashMap::new; - } - })); + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new), + Collections.singletonList(HashMap::new)); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade == metaData); assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData)); @@ -459,16 +407,12 @@ public UnaryOperator> getIndexTemplateMetaDat public void testIndexTemplateValidation() throws Exception { MetaData metaData = randomMetaData(); - MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList( - new Plugin() { - @Override - public UnaryOperator> getIndexTemplateMetaDataUpgrader() { - return customs -> { - throw new IllegalStateException("template is incompatible"); - }; - } - } - )); + MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( + Collections.emptyList(), + Collections.singletonList( + customs -> { + throw new IllegalStateException("template is incompatible"); + })); String message = expectThrows(IllegalStateException.class, () -> GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader)).getMessage(); assertThat(message, equalTo("template is incompatible")); @@ -479,10 +423,10 @@ public void testMultipleIndexTemplateUpgrade() throws Exception { final MetaData metaData; switch (randomIntBetween(0, 2)) { case 0: - metaData = randomMetaData(new CustomMetaData1("data1"), new CustomMetaData2("data2")); + metaData = randomMetaDataWithIndexTemplates("template1", "template2"); break; case 1: - metaData = randomMetaData(randomBoolean() ? new CustomMetaData1("data1") : new CustomMetaData2("data2")); + metaData = randomMetaDataWithIndexTemplates(randomBoolean() ? "template1" : "template2"); break; case 2: metaData = randomMetaData(); @@ -491,34 +435,28 @@ public void testMultipleIndexTemplateUpgrade() throws Exception { throw new IllegalStateException("should never happen"); } MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( + Collections.emptyList(), Arrays.asList( - new Plugin() { - @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return - customs -> { - customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1")); - return customs; - }; - } + indexTemplateMetaDatas -> { + indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1").settings( + Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build()).build()); + return indexTemplateMetaDatas; + }, - new Plugin() { - @Override - public UnaryOperator> getCustomMetaDataUpgrader() { - return customs -> { - customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2")); - return customs; - }; - } - }) - ); + indexTemplateMetaDatas -> { + indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2").settings( + Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build()); + return indexTemplateMetaDatas; + + } + )); MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader); assertTrue(upgrade != metaData); assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData)); - assertNotNull(upgrade.custom(CustomMetaData1.TYPE)); - assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1")); - assertNotNull(upgrade.custom(CustomMetaData2.TYPE)); - assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData2.TYPE)).getData(), equalTo("modified_data2")); + assertNotNull(upgrade.templates().get("template1")); + assertThat(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(upgrade.templates().get("template1").settings()), equalTo(20)); + assertNotNull(upgrade.templates().get("template2")); + assertThat(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(upgrade.templates().get("template2").settings()), equalTo(10)); for (IndexMetaData indexMetaData : upgrade) { assertTrue(metaData.hasIndexMetaData(indexMetaData)); } From 9b28a7ecbb903c56ab1515a56d834bf207b447b5 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 3 May 2017 11:16:28 -0400 Subject: [PATCH 6/6] Update javadoc --- .../java/org/elasticsearch/plugins/Plugin.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index 65502a96c5c82..bf5bc49d279c0 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -155,9 +155,13 @@ public void onIndexModule(IndexModule indexModule) {} * Provides a function to modify global custom meta data on startup. *

    * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required. + *

    + * The order of custom meta data upgraders calls is undefined and can change between runs so, it is expected that + * plugins will modify only data owned by them to avoid conflicts. + *

    * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map. * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom} - * is unsupported + * is unsupported */ public UnaryOperator> getCustomMetaDataUpgrader() { return UnaryOperator.identity(); @@ -167,9 +171,13 @@ public UnaryOperator> getCustomMetaDataUpgrader() { * Provides a function to modify index template meta data on startup. *

    * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required. + *

    + * The order of the template upgrader calls is undefined and can change between runs so, it is expected that + * plugins will modify only templates owned by them to avoid conflicts. + *

    * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map. * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData} - * cannot be upgraded + * cannot be upgraded */ public UnaryOperator> getIndexTemplateMetaDataUpgrader() { return UnaryOperator.identity(); @@ -179,6 +187,10 @@ public UnaryOperator> getIndexTemplateMetaDat * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time. *

    * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required. + *

    + * The order of the index upgrader calls for the same index is undefined and can change between runs so, it is expected that + * plugins will modify only indices owned by them to avoid conflicts. + *

    * @return Never {@code null}. The same or upgraded {@code IndexMetaData}. * @throws IllegalStateException if the node should not start because the index is unsupported */