diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 4cf74fbf865cc..8360797d66021 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -179,8 +179,12 @@ public void onFailure(String source, Exception e) { @Override public ClusterState execute(final ClusterState currentState) { - ClusterState clusterState = - updater.updateSettings(currentState, request.transientSettings(), request.persistentSettings(), logger); + final ClusterState clusterState = + updater.updateSettings( + currentState, + clusterSettings.upgradeSettings(request.transientSettings()), + clusterSettings.upgradeSettings(request.persistentSettings()), + logger); changed = clusterState != currentState; return clusterState; } diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java index ba18105e3f1ca..39829615fb3fe 100644 --- a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -19,7 +19,6 @@ package org.elasticsearch.client.transport; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionModule; @@ -44,6 +43,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.node.InternalSettingsPreparer; @@ -146,7 +146,8 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings for (final ExecutorBuilder builder : threadPool.builders()) { additionalSettings.addAll(builder.getRegisteredSettings()); } - SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter); + SettingsModule settingsModule = + new SettingsModule(settings, additionalSettings, additionalSettingsFilter, Collections.emptySet()); SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class)); IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); diff --git a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java index a77d739ffe0b4..e25d954aa4f1c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.regex.Regex; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -38,6 +39,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -52,14 +54,29 @@ public abstract class AbstractScopedSettings extends AbstractComponent { private final List> settingUpdaters = new CopyOnWriteArrayList<>(); private final Map> complexMatchers; private final Map> keySettings; + private final Map, Function, Map.Entry>> settingUpgraders; private final Setting.Property scope; private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$"); private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$"); private static final Pattern AFFIX_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+[*](?:[.][-\\w]+)+$"); - protected AbstractScopedSettings(Settings settings, Set> settingsSet, Setting.Property scope) { + protected AbstractScopedSettings( + final Settings settings, + final Set> settingsSet, + final Set> settingUpgraders, + final Setting.Property scope) { super(settings); this.lastSettingsApplied = Settings.EMPTY; + + this.settingUpgraders = + Collections.unmodifiableMap( + settingUpgraders + .stream() + .collect( + Collectors.toMap( + SettingUpgrader::getSetting, + u -> e -> new AbstractMap.SimpleEntry<>(u.getKey(e.getKey()), u.getValue(e.getValue()))))); + this.scope = scope; Map> complexMatchers = new HashMap<>(); Map> keySettings = new HashMap<>(); @@ -97,6 +114,7 @@ protected AbstractScopedSettings(Settings nodeSettings, Settings scopeSettings, this.scope = other.scope; complexMatchers = other.complexMatchers; keySettings = other.keySettings; + settingUpgraders = Collections.unmodifiableMap(new HashMap<>(other.settingUpgraders)); settingUpdaters.addAll(other.settingUpdaters); } @@ -757,6 +775,32 @@ private static Setting findOverlappingSetting(Setting newSetting, Map setting = getRaw(key); + final Function, Map.Entry> upgrader = settingUpgraders.get(setting); + if (upgrader == null) { + // the setting does not have an upgrader, copy the setting + builder.copy(key, settings); + } else { + // the setting has an upgrader, so mark that we have changed a setting and apply the upgrade logic + changed = true; + final Map.Entry upgrade = upgrader.apply(new Entry(key, settings)); + builder.put(upgrade.getKey(), upgrade.getValue()); + } + } + // we only return a new instance if there was an upgrade + return changed ? builder.build() : settings; + } + /** * Archives invalid or unknown settings. Any setting that is not recognized or fails validation * will be archived. This means the setting is prefixed with {@value ARCHIVED_SETTINGS_PREFIX} @@ -847,4 +891,5 @@ public String setValue(String value) { public boolean isPrivateSetting(String key) { return false; } + } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 10787140bdec8..cb369d6cfda02 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -100,6 +100,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.function.Predicate; @@ -107,8 +108,13 @@ * Encapsulates all valid cluster level settings. */ public final class ClusterSettings extends AbstractScopedSettings { - public ClusterSettings(Settings nodeSettings, Set> settingsSet) { - super(nodeSettings, settingsSet, Property.NodeScope); + public ClusterSettings(final Settings nodeSettings, final Set> settingsSet) { + this(nodeSettings, settingsSet, Collections.emptySet()); + } + + public ClusterSettings( + final Settings nodeSettings, final Set> settingsSet, final Set> settingUpgraders) { + super(nodeSettings, settingsSet, settingUpgraders, Property.NodeScope); addSettingsUpdater(new LoggingSettingUpdater(nodeSettings)); } @@ -436,4 +442,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndexGraveyard.SETTING_MAX_TOMBSTONES, EnableAssignmentDecider.CLUSTER_TASKS_ALLOCATION_ENABLE_SETTING ))); + + public static List> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList(); + } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 2116d1eff7510..ae8529af5b53e 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -178,7 +178,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS); public IndexScopedSettings(Settings settings, Set> settingsSet) { - super(settings, settingsSet, Property.IndexScope); + super(settings, settingsSet, Collections.emptySet(), Property.IndexScope); } private IndexScopedSettings(Settings settings, IndexScopedSettings other, IndexMetaData metaData) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java b/server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java new file mode 100644 index 0000000000000..91f2bead300d3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +/** + * Represents the logic to upgrade a setting. + * + * @param the type of the underlying setting + */ +public interface SettingUpgrader { + + /** + * The setting upgraded by this upgrader. + * + * @return the setting + */ + Setting getSetting(); + + /** + * The logic to upgrade the setting key, for example by mapping the old setting key to the new setting key. + * + * @param key the setting key to upgrade + * @return the upgraded setting key + */ + String getKey(String key); + + /** + * The logic to upgrade the setting value. + * + * @param value the setting value to upgrade + * @return the upgraded setting value + */ + default String getValue(final String value) { + return value; + } + +} diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java index 67037b3708bee..1eca3eb415f12 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java @@ -54,10 +54,14 @@ public class SettingsModule implements Module { private final SettingsFilter settingsFilter; public SettingsModule(Settings settings, Setting... additionalSettings) { - this(settings, Arrays.asList(additionalSettings), Collections.emptyList()); + this(settings, Arrays.asList(additionalSettings), Collections.emptyList(), Collections.emptySet()); } - public SettingsModule(Settings settings, List> additionalSettings, List settingsFilter) { + public SettingsModule( + Settings settings, + List> additionalSettings, + List settingsFilter, + Set> settingUpgraders) { logger = Loggers.getLogger(getClass(), settings); this.settings = settings; for (Setting setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) { @@ -70,12 +74,22 @@ public SettingsModule(Settings settings, List> additionalSettings, Li for (Setting setting : additionalSettings) { registerSetting(setting); } - for (String filter : settingsFilter) { registerSettingsFilter(filter); } + final Set> clusterSettingUpgraders = new HashSet<>(); + for (final SettingUpgrader settingUpgrader : ClusterSettings.BUILT_IN_SETTING_UPGRADERS) { + assert settingUpgrader.getSetting().hasNodeScope() : settingUpgrader.getSetting().getKey(); + final boolean added = clusterSettingUpgraders.add(settingUpgrader); + assert added : settingUpgrader.getSetting().getKey(); + } + for (final SettingUpgrader settingUpgrader : settingUpgraders) { + assert settingUpgrader.getSetting().hasNodeScope() : settingUpgrader.getSetting().getKey(); + final boolean added = clusterSettingUpgraders.add(settingUpgrader); + assert added : settingUpgrader.getSetting().getKey(); + } this.indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values())); - this.clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values())); + this.clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()), clusterSettingUpgraders); Settings indexSettings = settings.filter((s) -> (s.startsWith("index.") && // special case - we want to get Did you mean indices.query.bool.max_clause_count // which means we need to by-pass this check for this setting @@ -205,4 +219,5 @@ public ClusterSettings getClusterSettings() { public SettingsFilter getSettingsFilter() { return settingsFilter; } + } diff --git a/server/src/main/java/org/elasticsearch/gateway/Gateway.java b/server/src/main/java/org/elasticsearch/gateway/Gateway.java index d2261e5d1b421..77d2c553c2c51 100644 --- a/server/src/main/java/org/elasticsearch/gateway/Gateway.java +++ b/server/src/main/java/org/elasticsearch/gateway/Gateway.java @@ -137,20 +137,25 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t } } } + final ClusterState.Builder builder = upgradeAndArchiveUnknownOrInvalidSettings(metaDataBuilder); + listener.onSuccess(builder.build()); + } + + ClusterState.Builder upgradeAndArchiveUnknownOrInvalidSettings(MetaData.Builder metaDataBuilder) { final ClusterSettings clusterSettings = clusterService.getClusterSettings(); metaDataBuilder.persistentSettings( clusterSettings.archiveUnknownOrInvalidSettings( - metaDataBuilder.persistentSettings(), + clusterSettings.upgradeSettings(metaDataBuilder.persistentSettings()), e -> logUnknownSetting("persistent", e), (e, ex) -> logInvalidSetting("persistent", e, ex))); metaDataBuilder.transientSettings( clusterSettings.archiveUnknownOrInvalidSettings( - metaDataBuilder.transientSettings(), + clusterSettings.upgradeSettings(metaDataBuilder.transientSettings()), e -> logUnknownSetting("transient", e), (e, ex) -> logInvalidSetting("transient", e, ex))); ClusterState.Builder builder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)); builder.metaData(metaDataBuilder); - listener.onSuccess(builder.build()); + return builder; } private void logUnknownSetting(String settingType, Map.Entry e) { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index c2ef6d12331fe..67c3894ddf40a 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -73,6 +73,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.SettingUpgrader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -151,6 +152,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import javax.net.ssl.SNIHostName; + import java.io.BufferedWriter; import java.io.Closeable; import java.io.IOException; @@ -360,7 +362,15 @@ protected Node( AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class)); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already - final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter); + + final Set> settingsUpgraders = pluginsService.filterPlugins(Plugin.class) + .stream() + .map(Plugin::getSettingUpgraders) + .flatMap(List::stream) + .collect(Collectors.toSet()); + + final SettingsModule settingsModule = + new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter, settingsUpgraders); scriptModule.registerClusterSettingsListeners(settingsModule.getClusterSettings()); resourcesToClose.add(resourceWatcherService); final NetworkService networkService = new NetworkService( diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index 65d47682a95c0..faef27207e13a 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.SettingUpgrader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -172,6 +173,15 @@ public void onIndexModule(IndexModule indexModule) {} */ public List getSettingsFilter() { return Collections.emptyList(); } + /** + * Get the setting upgraders provided by this plugin. + * + * @return the settings upgraders + */ + public List> getSettingUpgraders() { + return Collections.emptyList(); + } + /** * Provides a function to modify global custom meta data on startup. *

diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java index b67c2e2954d04..9bf4d9d32f622 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java @@ -40,9 +40,11 @@ import org.junit.After; import org.junit.Before; -import java.util.Collections; import java.util.concurrent.TimeUnit; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; + public class GetIndexActionTests extends ESSingleNodeTestCase { private TransportService transportService; @@ -58,14 +60,14 @@ public class GetIndexActionTests extends ESSingleNodeTestCase { public void setUp() throws Exception { super.setUp(); - settingsFilter = new SettingsModule(Settings.EMPTY, Collections.emptyList(), Collections.emptyList()).getSettingsFilter(); + settingsFilter = new SettingsModule(Settings.EMPTY, emptyList(), emptyList(), emptySet()).getSettingsFilter(); threadPool = new TestThreadPool("GetIndexActionTests"); clusterService = getInstanceFromNode(ClusterService.class); indicesService = getInstanceFromNode(IndicesService.class); CapturingTransport capturingTransport = new CapturingTransport(); transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> clusterService.localNode(), null, Collections.emptySet()); + boundAddress -> clusterService.localNode(), null, emptySet()); transportService.start(); transportService.acceptIncomingRequests(); getIndexAction = new GetIndexActionTests.TestTransportGetIndexAction(); @@ -106,7 +108,7 @@ class TestTransportGetIndexAction extends TransportGetIndexAction { TestTransportGetIndexAction() { super(Settings.EMPTY, GetIndexActionTests.this.transportService, GetIndexActionTests.this.clusterService, - GetIndexActionTests.this.threadPool, settingsFilter, new ActionFilters(Collections.emptySet()), + GetIndexActionTests.this.threadPool, settingsFilter, new ActionFilters(emptySet()), new GetIndexActionTests.Resolver(Settings.EMPTY), indicesService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java index 03ccebba10dbd..85b85cf9e1469 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java @@ -42,6 +42,8 @@ import java.util.Collections; import java.util.concurrent.TimeUnit; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; public class GetSettingsActionTests extends ESTestCase { @@ -71,7 +73,7 @@ protected void masterOperation(GetSettingsRequest request, ClusterState state, A public void setUp() throws Exception { super.setUp(); - settingsFilter = new SettingsModule(Settings.EMPTY, Collections.emptyList(), Collections.emptyList()).getSettingsFilter(); + settingsFilter = new SettingsModule(Settings.EMPTY, emptyList(), emptyList(), emptySet()).getSettingsFilter(); threadPool = new TestThreadPool("GetSettingsActionTests"); clusterService = createClusterService(threadPool); CapturingTransport capturingTransport = new CapturingTransport(); diff --git a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java index f0f8b6c417f2f..0ee1d2e9c4a80 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -53,6 +54,7 @@ import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.sameInstance; public class ScopedSettingsTests extends ESTestCase { @@ -1045,4 +1047,128 @@ public void testPrivateIndexSettingsSkipValidation() { indexScopedSettings.validate(settings, false, /* validateInternalOrPrivateIndex */ false); } + public void testUpgradeSetting() { + final Setting oldSetting = Setting.simpleString("foo.old", Property.NodeScope); + final Setting newSetting = Setting.simpleString("foo.new", Property.NodeScope); + final Setting remainingSetting = Setting.simpleString("foo.remaining", Property.NodeScope); + + final AbstractScopedSettings service = + new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(oldSetting, newSetting, remainingSetting)), + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + + })); + + final Settings settings = + Settings.builder() + .put("foo.old", randomAlphaOfLength(8)) + .put("foo.remaining", randomAlphaOfLength(8)) + .build(); + final Settings upgradedSettings = service.upgradeSettings(settings); + assertFalse(oldSetting.exists(upgradedSettings)); + assertTrue(newSetting.exists(upgradedSettings)); + assertThat(newSetting.get(upgradedSettings), equalTo("new." + oldSetting.get(settings))); + assertTrue(remainingSetting.exists(upgradedSettings)); + assertThat(remainingSetting.get(upgradedSettings), equalTo(remainingSetting.get(settings))); + } + + public void testUpgradeSettingsNoChangesPreservesInstance() { + final Setting oldSetting = Setting.simpleString("foo.old", Property.NodeScope); + final Setting newSetting = Setting.simpleString("foo.new", Property.NodeScope); + final Setting remainingSetting = Setting.simpleString("foo.remaining", Property.NodeScope); + + final AbstractScopedSettings service = + new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(oldSetting, newSetting, remainingSetting)), + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + })); + + final Settings settings = Settings.builder().put("foo.remaining", randomAlphaOfLength(8)).build(); + final Settings upgradedSettings = service.upgradeSettings(settings); + assertThat(upgradedSettings, sameInstance(settings)); + } + + public void testUpgradeComplexSetting() { + final Setting.AffixSetting oldSetting = + Setting.affixKeySetting("foo.old.", "suffix", key -> Setting.simpleString(key, Property.NodeScope)); + final Setting.AffixSetting newSetting = + Setting.affixKeySetting("foo.new.", "suffix", key -> Setting.simpleString(key, Property.NodeScope)); + final Setting.AffixSetting remainingSetting = + Setting.affixKeySetting("foo.remaining.", "suffix", key -> Setting.simpleString(key, Property.NodeScope)); + + final AbstractScopedSettings service = + new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(oldSetting, newSetting, remainingSetting)), + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return key.replaceFirst("^foo\\.old", "foo\\.new"); + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + + })); + + final int count = randomIntBetween(1, 8); + final List concretes = new ArrayList<>(count); + final Settings.Builder builder = Settings.builder(); + for (int i = 0; i < count; i++) { + final String concrete = randomAlphaOfLength(8); + concretes.add(concrete); + builder.put("foo.old." + concrete + ".suffix", randomAlphaOfLength(8)); + builder.put("foo.remaining." + concrete + ".suffix", randomAlphaOfLength(8)); + } + final Settings settings = builder.build(); + final Settings upgradedSettings = service.upgradeSettings(settings); + for (final String concrete : concretes) { + assertFalse(oldSetting.getConcreteSettingForNamespace(concrete).exists(upgradedSettings)); + assertTrue(newSetting.getConcreteSettingForNamespace(concrete).exists(upgradedSettings)); + assertThat( + newSetting.getConcreteSettingForNamespace(concrete).get(upgradedSettings), + equalTo("new." + oldSetting.getConcreteSettingForNamespace(concrete).get(settings))); + assertTrue(remainingSetting.getConcreteSettingForNamespace(concrete).exists(upgradedSettings)); + assertThat( + remainingSetting.getConcreteSettingForNamespace(concrete).get(upgradedSettings), + equalTo(remainingSetting.getConcreteSettingForNamespace(concrete).get(settings))); + } + } + } diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java index 6a2be8217a661..c6182eac8f680 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java @@ -24,6 +24,7 @@ import java.util.Arrays; +import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.containsString; public class SettingsModuleTests extends ModuleTestCase { @@ -103,14 +104,14 @@ public void testRegisterSettingsFilter() { try { new SettingsModule(settings, Arrays.asList(Setting.boolSetting("foo.bar", true, Property.NodeScope), Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered), - Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*", "bar.foo")); + Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*", "bar.foo"), emptySet()); fail(); } catch (IllegalArgumentException ex) { assertEquals("filter [bar.foo] has already been registered", ex.getMessage()); } SettingsModule module = new SettingsModule(settings, Arrays.asList(Setting.boolSetting("foo.bar", true, Property.NodeScope), Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered), - Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*")); + Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*"), emptySet()); assertInstanceBinding(module, Settings.class, (s) -> s == settings); assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).size() == 1); assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).keySet().contains("bar.baz")); diff --git a/server/src/test/java/org/elasticsearch/common/settings/UpgradeSettingsIT.java b/server/src/test/java/org/elasticsearch/common/settings/UpgradeSettingsIT.java new file mode 100644 index 0000000000000..839b96e641870 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/settings/UpgradeSettingsIT.java @@ -0,0 +1,125 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.junit.After; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import static org.hamcrest.Matchers.equalTo; + +public class UpgradeSettingsIT extends ESSingleNodeTestCase { + + @After + public void cleanup() throws Exception { + client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("*")) + .setTransientSettings(Settings.builder().putNull("*")) + .get(); + } + + @Override + protected Collection> getPlugins() { + return Collections.singletonList(UpgradeSettingsPlugin.class); + } + + public static class UpgradeSettingsPlugin extends Plugin { + + static final Setting oldSetting = Setting.simpleString("foo.old", Setting.Property.Dynamic, Setting.Property.NodeScope); + static final Setting newSetting = Setting.simpleString("foo.new", Setting.Property.Dynamic, Setting.Property.NodeScope); + + public UpgradeSettingsPlugin(){ + + } + + @Override + public List> getSettings() { + return Arrays.asList(oldSetting, newSetting); + } + + @Override + public List> getSettingUpgraders() { + return Collections.singletonList(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + }); + } + } + + public void testUpgradePersistentSettingsOnUpdate() { + runUpgradeSettingsOnUpdateTest((settings, builder) -> builder.setPersistentSettings(settings), MetaData::persistentSettings); + } + + public void testUpgradeTransientSettingsOnUpdate() { + runUpgradeSettingsOnUpdateTest((settings, builder) -> builder.setTransientSettings(settings), MetaData::transientSettings); + } + + private void runUpgradeSettingsOnUpdateTest( + final BiConsumer consumer, + final Function settingsFunction) { + final String value = randomAlphaOfLength(8); + final ClusterUpdateSettingsRequestBuilder builder = + client() + .admin() + .cluster() + .prepareUpdateSettings(); + consumer.accept(Settings.builder().put("foo.old", value).build(), builder); + builder.get(); + + final ClusterStateResponse response = client() + .admin() + .cluster() + .prepareState() + .clear() + .setMetaData(true) + .get(); + + assertFalse(UpgradeSettingsPlugin.oldSetting.exists(settingsFunction.apply(response.getState().metaData()))); + assertTrue(UpgradeSettingsPlugin.newSetting.exists(settingsFunction.apply(response.getState().metaData()))); + assertThat(UpgradeSettingsPlugin.newSetting.get(settingsFunction.apply(response.getState().metaData())), equalTo("new." + value)); + } + +} diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayTests.java new file mode 100644 index 0000000000000..457b3a14ebf4a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayTests.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gateway; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.SettingUpgrader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; + +public class GatewayTests extends ESTestCase { + + public void testUpgradePersistentSettings() { + runUpgradeSettings(MetaData.Builder::persistentSettings, MetaData::persistentSettings); + } + + public void testUpgradeTransientSettings() { + runUpgradeSettings(MetaData.Builder::transientSettings, MetaData::transientSettings); + } + + private void runUpgradeSettings( + final BiConsumer applySettingsToBuilder, final Function metaDataSettings) { + final Setting oldSetting = Setting.simpleString("foo.old", Setting.Property.Dynamic, Setting.Property.NodeScope); + final Setting newSetting = Setting.simpleString("foo.new", Setting.Property.Dynamic, Setting.Property.NodeScope); + final Set> settingsSet = + Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of(oldSetting, newSetting)).collect(Collectors.toSet()); + final ClusterSettings clusterSettings = new ClusterSettings( + Settings.EMPTY, + settingsSet, + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + + })); + final ClusterService clusterService = new ClusterService(Settings.EMPTY, clusterSettings, null); + final Gateway gateway = new Gateway(Settings.EMPTY, clusterService, null, null); + final MetaData.Builder builder = MetaData.builder(); + final Settings settings = Settings.builder().put("foo.old", randomAlphaOfLength(8)).build(); + applySettingsToBuilder.accept(builder, settings); + final ClusterState state = gateway.upgradeAndArchiveUnknownOrInvalidSettings(builder).build(); + assertFalse(oldSetting.exists(metaDataSettings.apply(state.metaData()))); + assertTrue(newSetting.exists(metaDataSettings.apply(state.metaData()))); + assertThat(newSetting.get(metaDataSettings.apply(state.metaData())), equalTo("new." + oldSetting.get(settings))); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index cc3902893411a..60f93f8ea30fe 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -340,7 +340,8 @@ private static class ServiceHolder implements Closeable { clientInvocationHandler); ScriptModule scriptModule = createScriptModule(pluginsService.filterPlugins(ScriptPlugin.class)); List> additionalSettings = pluginsService.getPluginSettings(); - SettingsModule settingsModule = new SettingsModule(nodeSettings, additionalSettings, pluginsService.getPluginSettingsFilter()); + SettingsModule settingsModule = + new SettingsModule(nodeSettings, additionalSettings, pluginsService.getPluginSettingsFilter(), Collections.emptySet()); searchModule = new SearchModule(nodeSettings, false, pluginsService.filterPlugins(SearchPlugin.class)); IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class)); List entries = new ArrayList<>(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java index f7fabab2799af..daf1775a80a52 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.core.security.SecurityExtension; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -181,7 +182,8 @@ private static void validateRealm(String name, String type, Settings settings, S settingSet.add(TYPE_SETTING); settingSet.add(ENABLED_SETTING); settingSet.add(ORDER_SETTING); - final AbstractScopedSettings validator = new AbstractScopedSettings(settings, settingSet, Setting.Property.NodeScope) { }; + final AbstractScopedSettings validator = + new AbstractScopedSettings(settings, settingSet, Collections.emptySet(), Setting.Property.NodeScope) { }; try { validator.validate(settings, false); } catch (RuntimeException e) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java index 1886dd4249b14..3bf3bb4dc8641 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java @@ -20,7 +20,9 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; + import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -121,7 +123,7 @@ public void testFiltering() throws Exception { List settingsFilterList = new ArrayList<>(); settingsFilterList.addAll(securityPlugin.getSettingsFilter()); // custom settings, potentially added by a plugin - SettingsModule settingsModule = new SettingsModule(settings, settingList, settingsFilterList); + SettingsModule settingsModule = new SettingsModule(settings, settingList, settingsFilterList, Collections.emptySet()); Injector injector = Guice.createInjector(settingsModule); SettingsFilter settingsFilter = injector.getInstance(SettingsFilter.class);