From 6bb817004b9d3f4cc9e723f0df50b5a2f1cdc064 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 9 Sep 2018 20:49:19 -0400 Subject: [PATCH] Add infrastructure to upgrade settings (#33536) In some cases we want to deprecate a setting, and then automatically upgrade uses of that setting to a replacement setting. This commit adds infrastructure for this so that we can upgrade settings when recovering the cluster state, as well as when such settings are dynamically applied on cluster update settings requests. This commit only focuses on cluster settings, index settings can build on this infrastructure in a follow-up. --- .../TransportClusterUpdateSettingsAction.java | 8 +- .../client/transport/TransportClient.java | 5 +- .../settings/AbstractScopedSettings.java | 47 ++++++- .../common/settings/ClusterSettings.java | 13 +- .../common/settings/IndexScopedSettings.java | 2 +- .../common/settings/SettingUpgrader.java | 54 ++++++++ .../common/settings/SettingsModule.java | 23 +++- .../org/elasticsearch/gateway/Gateway.java | 11 +- .../java/org/elasticsearch/node/Node.java | 12 +- .../org/elasticsearch/plugins/Plugin.java | 10 ++ .../indices/get/GetIndexActionTests.java | 10 +- .../settings/get/GetSettingsActionTests.java | 4 +- .../common/settings/ScopedSettingsTests.java | 126 ++++++++++++++++++ .../common/settings/SettingsModuleTests.java | 5 +- .../common/settings/UpgradeSettingsIT.java | 125 +++++++++++++++++ .../elasticsearch/gateway/GatewayTests.java | 90 +++++++++++++ .../test/AbstractBuilderTestCase.java | 3 +- .../core/security/authc/RealmSettings.java | 4 +- .../test/SettingsFilterTests.java | 4 +- 19 files changed, 530 insertions(+), 26 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java create mode 100644 server/src/test/java/org/elasticsearch/common/settings/UpgradeSettingsIT.java create mode 100644 server/src/test/java/org/elasticsearch/gateway/GatewayTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 4cf74fbf865cc..8360797d66021 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -179,8 +179,12 @@ public void onFailure(String source, Exception e) { @Override public ClusterState execute(final ClusterState currentState) { - ClusterState clusterState = - updater.updateSettings(currentState, request.transientSettings(), request.persistentSettings(), logger); + final ClusterState clusterState = + updater.updateSettings( + currentState, + clusterSettings.upgradeSettings(request.transientSettings()), + clusterSettings.upgradeSettings(request.persistentSettings()), + logger); changed = clusterState != currentState; return clusterState; } diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java index ba18105e3f1ca..39829615fb3fe 100644 --- a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -19,7 +19,6 @@ package org.elasticsearch.client.transport; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionModule; @@ -44,6 +43,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.node.InternalSettingsPreparer; @@ -146,7 +146,8 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings for (final ExecutorBuilder builder : threadPool.builders()) { additionalSettings.addAll(builder.getRegisteredSettings()); } - SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter); + SettingsModule settingsModule = + new SettingsModule(settings, additionalSettings, additionalSettingsFilter, Collections.emptySet()); SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class)); IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); diff --git a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java index a77d739ffe0b4..e25d954aa4f1c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.regex.Regex; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -38,6 +39,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -52,14 +54,29 @@ public abstract class AbstractScopedSettings extends AbstractComponent { private final List> settingUpdaters = new CopyOnWriteArrayList<>(); private final Map> complexMatchers; private final Map> keySettings; + private final Map, Function, Map.Entry>> settingUpgraders; private final Setting.Property scope; private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$"); private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$"); private static final Pattern AFFIX_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+[*](?:[.][-\\w]+)+$"); - protected AbstractScopedSettings(Settings settings, Set> settingsSet, Setting.Property scope) { + protected AbstractScopedSettings( + final Settings settings, + final Set> settingsSet, + final Set> settingUpgraders, + final Setting.Property scope) { super(settings); this.lastSettingsApplied = Settings.EMPTY; + + this.settingUpgraders = + Collections.unmodifiableMap( + settingUpgraders + .stream() + .collect( + Collectors.toMap( + SettingUpgrader::getSetting, + u -> e -> new AbstractMap.SimpleEntry<>(u.getKey(e.getKey()), u.getValue(e.getValue()))))); + this.scope = scope; Map> complexMatchers = new HashMap<>(); Map> keySettings = new HashMap<>(); @@ -97,6 +114,7 @@ protected AbstractScopedSettings(Settings nodeSettings, Settings scopeSettings, this.scope = other.scope; complexMatchers = other.complexMatchers; keySettings = other.keySettings; + settingUpgraders = Collections.unmodifiableMap(new HashMap<>(other.settingUpgraders)); settingUpdaters.addAll(other.settingUpdaters); } @@ -757,6 +775,32 @@ private static Setting findOverlappingSetting(Setting newSetting, Map setting = getRaw(key); + final Function, Map.Entry> upgrader = settingUpgraders.get(setting); + if (upgrader == null) { + // the setting does not have an upgrader, copy the setting + builder.copy(key, settings); + } else { + // the setting has an upgrader, so mark that we have changed a setting and apply the upgrade logic + changed = true; + final Map.Entry upgrade = upgrader.apply(new Entry(key, settings)); + builder.put(upgrade.getKey(), upgrade.getValue()); + } + } + // we only return a new instance if there was an upgrade + return changed ? builder.build() : settings; + } + /** * Archives invalid or unknown settings. Any setting that is not recognized or fails validation * will be archived. This means the setting is prefixed with {@value ARCHIVED_SETTINGS_PREFIX} @@ -847,4 +891,5 @@ public String setValue(String value) { public boolean isPrivateSetting(String key) { return false; } + } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 10787140bdec8..cb369d6cfda02 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -100,6 +100,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.function.Predicate; @@ -107,8 +108,13 @@ * Encapsulates all valid cluster level settings. */ public final class ClusterSettings extends AbstractScopedSettings { - public ClusterSettings(Settings nodeSettings, Set> settingsSet) { - super(nodeSettings, settingsSet, Property.NodeScope); + public ClusterSettings(final Settings nodeSettings, final Set> settingsSet) { + this(nodeSettings, settingsSet, Collections.emptySet()); + } + + public ClusterSettings( + final Settings nodeSettings, final Set> settingsSet, final Set> settingUpgraders) { + super(nodeSettings, settingsSet, settingUpgraders, Property.NodeScope); addSettingsUpdater(new LoggingSettingUpdater(nodeSettings)); } @@ -436,4 +442,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndexGraveyard.SETTING_MAX_TOMBSTONES, EnableAssignmentDecider.CLUSTER_TASKS_ALLOCATION_ENABLE_SETTING ))); + + public static List> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList(); + } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 2116d1eff7510..ae8529af5b53e 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -178,7 +178,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS); public IndexScopedSettings(Settings settings, Set> settingsSet) { - super(settings, settingsSet, Property.IndexScope); + super(settings, settingsSet, Collections.emptySet(), Property.IndexScope); } private IndexScopedSettings(Settings settings, IndexScopedSettings other, IndexMetaData metaData) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java b/server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java new file mode 100644 index 0000000000000..91f2bead300d3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +/** + * Represents the logic to upgrade a setting. + * + * @param the type of the underlying setting + */ +public interface SettingUpgrader { + + /** + * The setting upgraded by this upgrader. + * + * @return the setting + */ + Setting getSetting(); + + /** + * The logic to upgrade the setting key, for example by mapping the old setting key to the new setting key. + * + * @param key the setting key to upgrade + * @return the upgraded setting key + */ + String getKey(String key); + + /** + * The logic to upgrade the setting value. + * + * @param value the setting value to upgrade + * @return the upgraded setting value + */ + default String getValue(final String value) { + return value; + } + +} diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java index 67037b3708bee..1eca3eb415f12 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java @@ -54,10 +54,14 @@ public class SettingsModule implements Module { private final SettingsFilter settingsFilter; public SettingsModule(Settings settings, Setting... additionalSettings) { - this(settings, Arrays.asList(additionalSettings), Collections.emptyList()); + this(settings, Arrays.asList(additionalSettings), Collections.emptyList(), Collections.emptySet()); } - public SettingsModule(Settings settings, List> additionalSettings, List settingsFilter) { + public SettingsModule( + Settings settings, + List> additionalSettings, + List settingsFilter, + Set> settingUpgraders) { logger = Loggers.getLogger(getClass(), settings); this.settings = settings; for (Setting setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) { @@ -70,12 +74,22 @@ public SettingsModule(Settings settings, List> additionalSettings, Li for (Setting setting : additionalSettings) { registerSetting(setting); } - for (String filter : settingsFilter) { registerSettingsFilter(filter); } + final Set> clusterSettingUpgraders = new HashSet<>(); + for (final SettingUpgrader settingUpgrader : ClusterSettings.BUILT_IN_SETTING_UPGRADERS) { + assert settingUpgrader.getSetting().hasNodeScope() : settingUpgrader.getSetting().getKey(); + final boolean added = clusterSettingUpgraders.add(settingUpgrader); + assert added : settingUpgrader.getSetting().getKey(); + } + for (final SettingUpgrader settingUpgrader : settingUpgraders) { + assert settingUpgrader.getSetting().hasNodeScope() : settingUpgrader.getSetting().getKey(); + final boolean added = clusterSettingUpgraders.add(settingUpgrader); + assert added : settingUpgrader.getSetting().getKey(); + } this.indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values())); - this.clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values())); + this.clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()), clusterSettingUpgraders); Settings indexSettings = settings.filter((s) -> (s.startsWith("index.") && // special case - we want to get Did you mean indices.query.bool.max_clause_count // which means we need to by-pass this check for this setting @@ -205,4 +219,5 @@ public ClusterSettings getClusterSettings() { public SettingsFilter getSettingsFilter() { return settingsFilter; } + } diff --git a/server/src/main/java/org/elasticsearch/gateway/Gateway.java b/server/src/main/java/org/elasticsearch/gateway/Gateway.java index d2261e5d1b421..77d2c553c2c51 100644 --- a/server/src/main/java/org/elasticsearch/gateway/Gateway.java +++ b/server/src/main/java/org/elasticsearch/gateway/Gateway.java @@ -137,20 +137,25 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t } } } + final ClusterState.Builder builder = upgradeAndArchiveUnknownOrInvalidSettings(metaDataBuilder); + listener.onSuccess(builder.build()); + } + + ClusterState.Builder upgradeAndArchiveUnknownOrInvalidSettings(MetaData.Builder metaDataBuilder) { final ClusterSettings clusterSettings = clusterService.getClusterSettings(); metaDataBuilder.persistentSettings( clusterSettings.archiveUnknownOrInvalidSettings( - metaDataBuilder.persistentSettings(), + clusterSettings.upgradeSettings(metaDataBuilder.persistentSettings()), e -> logUnknownSetting("persistent", e), (e, ex) -> logInvalidSetting("persistent", e, ex))); metaDataBuilder.transientSettings( clusterSettings.archiveUnknownOrInvalidSettings( - metaDataBuilder.transientSettings(), + clusterSettings.upgradeSettings(metaDataBuilder.transientSettings()), e -> logUnknownSetting("transient", e), (e, ex) -> logInvalidSetting("transient", e, ex))); ClusterState.Builder builder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)); builder.metaData(metaDataBuilder); - listener.onSuccess(builder.build()); + return builder; } private void logUnknownSetting(String settingType, Map.Entry e) { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index c2ef6d12331fe..67c3894ddf40a 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -73,6 +73,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.SettingUpgrader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -151,6 +152,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import javax.net.ssl.SNIHostName; + import java.io.BufferedWriter; import java.io.Closeable; import java.io.IOException; @@ -360,7 +362,15 @@ protected Node( AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class)); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already - final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter); + + final Set> settingsUpgraders = pluginsService.filterPlugins(Plugin.class) + .stream() + .map(Plugin::getSettingUpgraders) + .flatMap(List::stream) + .collect(Collectors.toSet()); + + final SettingsModule settingsModule = + new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter, settingsUpgraders); scriptModule.registerClusterSettingsListeners(settingsModule.getClusterSettings()); resourcesToClose.add(resourceWatcherService); final NetworkService networkService = new NetworkService( diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index 65d47682a95c0..faef27207e13a 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.SettingUpgrader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -172,6 +173,15 @@ public void onIndexModule(IndexModule indexModule) {} */ public List getSettingsFilter() { return Collections.emptyList(); } + /** + * Get the setting upgraders provided by this plugin. + * + * @return the settings upgraders + */ + public List> getSettingUpgraders() { + return Collections.emptyList(); + } + /** * Provides a function to modify global custom meta data on startup. *

diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java index b67c2e2954d04..9bf4d9d32f622 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java @@ -40,9 +40,11 @@ import org.junit.After; import org.junit.Before; -import java.util.Collections; import java.util.concurrent.TimeUnit; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; + public class GetIndexActionTests extends ESSingleNodeTestCase { private TransportService transportService; @@ -58,14 +60,14 @@ public class GetIndexActionTests extends ESSingleNodeTestCase { public void setUp() throws Exception { super.setUp(); - settingsFilter = new SettingsModule(Settings.EMPTY, Collections.emptyList(), Collections.emptyList()).getSettingsFilter(); + settingsFilter = new SettingsModule(Settings.EMPTY, emptyList(), emptyList(), emptySet()).getSettingsFilter(); threadPool = new TestThreadPool("GetIndexActionTests"); clusterService = getInstanceFromNode(ClusterService.class); indicesService = getInstanceFromNode(IndicesService.class); CapturingTransport capturingTransport = new CapturingTransport(); transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> clusterService.localNode(), null, Collections.emptySet()); + boundAddress -> clusterService.localNode(), null, emptySet()); transportService.start(); transportService.acceptIncomingRequests(); getIndexAction = new GetIndexActionTests.TestTransportGetIndexAction(); @@ -106,7 +108,7 @@ class TestTransportGetIndexAction extends TransportGetIndexAction { TestTransportGetIndexAction() { super(Settings.EMPTY, GetIndexActionTests.this.transportService, GetIndexActionTests.this.clusterService, - GetIndexActionTests.this.threadPool, settingsFilter, new ActionFilters(Collections.emptySet()), + GetIndexActionTests.this.threadPool, settingsFilter, new ActionFilters(emptySet()), new GetIndexActionTests.Resolver(Settings.EMPTY), indicesService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java index 03ccebba10dbd..85b85cf9e1469 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java @@ -42,6 +42,8 @@ import java.util.Collections; import java.util.concurrent.TimeUnit; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; public class GetSettingsActionTests extends ESTestCase { @@ -71,7 +73,7 @@ protected void masterOperation(GetSettingsRequest request, ClusterState state, A public void setUp() throws Exception { super.setUp(); - settingsFilter = new SettingsModule(Settings.EMPTY, Collections.emptyList(), Collections.emptyList()).getSettingsFilter(); + settingsFilter = new SettingsModule(Settings.EMPTY, emptyList(), emptyList(), emptySet()).getSettingsFilter(); threadPool = new TestThreadPool("GetSettingsActionTests"); clusterService = createClusterService(threadPool); CapturingTransport capturingTransport = new CapturingTransport(); diff --git a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java index f0f8b6c417f2f..0ee1d2e9c4a80 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -53,6 +54,7 @@ import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.sameInstance; public class ScopedSettingsTests extends ESTestCase { @@ -1045,4 +1047,128 @@ public void testPrivateIndexSettingsSkipValidation() { indexScopedSettings.validate(settings, false, /* validateInternalOrPrivateIndex */ false); } + public void testUpgradeSetting() { + final Setting oldSetting = Setting.simpleString("foo.old", Property.NodeScope); + final Setting newSetting = Setting.simpleString("foo.new", Property.NodeScope); + final Setting remainingSetting = Setting.simpleString("foo.remaining", Property.NodeScope); + + final AbstractScopedSettings service = + new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(oldSetting, newSetting, remainingSetting)), + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + + })); + + final Settings settings = + Settings.builder() + .put("foo.old", randomAlphaOfLength(8)) + .put("foo.remaining", randomAlphaOfLength(8)) + .build(); + final Settings upgradedSettings = service.upgradeSettings(settings); + assertFalse(oldSetting.exists(upgradedSettings)); + assertTrue(newSetting.exists(upgradedSettings)); + assertThat(newSetting.get(upgradedSettings), equalTo("new." + oldSetting.get(settings))); + assertTrue(remainingSetting.exists(upgradedSettings)); + assertThat(remainingSetting.get(upgradedSettings), equalTo(remainingSetting.get(settings))); + } + + public void testUpgradeSettingsNoChangesPreservesInstance() { + final Setting oldSetting = Setting.simpleString("foo.old", Property.NodeScope); + final Setting newSetting = Setting.simpleString("foo.new", Property.NodeScope); + final Setting remainingSetting = Setting.simpleString("foo.remaining", Property.NodeScope); + + final AbstractScopedSettings service = + new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(oldSetting, newSetting, remainingSetting)), + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + })); + + final Settings settings = Settings.builder().put("foo.remaining", randomAlphaOfLength(8)).build(); + final Settings upgradedSettings = service.upgradeSettings(settings); + assertThat(upgradedSettings, sameInstance(settings)); + } + + public void testUpgradeComplexSetting() { + final Setting.AffixSetting oldSetting = + Setting.affixKeySetting("foo.old.", "suffix", key -> Setting.simpleString(key, Property.NodeScope)); + final Setting.AffixSetting newSetting = + Setting.affixKeySetting("foo.new.", "suffix", key -> Setting.simpleString(key, Property.NodeScope)); + final Setting.AffixSetting remainingSetting = + Setting.affixKeySetting("foo.remaining.", "suffix", key -> Setting.simpleString(key, Property.NodeScope)); + + final AbstractScopedSettings service = + new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(oldSetting, newSetting, remainingSetting)), + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return key.replaceFirst("^foo\\.old", "foo\\.new"); + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + + })); + + final int count = randomIntBetween(1, 8); + final List concretes = new ArrayList<>(count); + final Settings.Builder builder = Settings.builder(); + for (int i = 0; i < count; i++) { + final String concrete = randomAlphaOfLength(8); + concretes.add(concrete); + builder.put("foo.old." + concrete + ".suffix", randomAlphaOfLength(8)); + builder.put("foo.remaining." + concrete + ".suffix", randomAlphaOfLength(8)); + } + final Settings settings = builder.build(); + final Settings upgradedSettings = service.upgradeSettings(settings); + for (final String concrete : concretes) { + assertFalse(oldSetting.getConcreteSettingForNamespace(concrete).exists(upgradedSettings)); + assertTrue(newSetting.getConcreteSettingForNamespace(concrete).exists(upgradedSettings)); + assertThat( + newSetting.getConcreteSettingForNamespace(concrete).get(upgradedSettings), + equalTo("new." + oldSetting.getConcreteSettingForNamespace(concrete).get(settings))); + assertTrue(remainingSetting.getConcreteSettingForNamespace(concrete).exists(upgradedSettings)); + assertThat( + remainingSetting.getConcreteSettingForNamespace(concrete).get(upgradedSettings), + equalTo(remainingSetting.getConcreteSettingForNamespace(concrete).get(settings))); + } + } + } diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java index 6a2be8217a661..c6182eac8f680 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java @@ -24,6 +24,7 @@ import java.util.Arrays; +import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.containsString; public class SettingsModuleTests extends ModuleTestCase { @@ -103,14 +104,14 @@ public void testRegisterSettingsFilter() { try { new SettingsModule(settings, Arrays.asList(Setting.boolSetting("foo.bar", true, Property.NodeScope), Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered), - Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*", "bar.foo")); + Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*", "bar.foo"), emptySet()); fail(); } catch (IllegalArgumentException ex) { assertEquals("filter [bar.foo] has already been registered", ex.getMessage()); } SettingsModule module = new SettingsModule(settings, Arrays.asList(Setting.boolSetting("foo.bar", true, Property.NodeScope), Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered), - Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*")); + Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*"), emptySet()); assertInstanceBinding(module, Settings.class, (s) -> s == settings); assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).size() == 1); assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).keySet().contains("bar.baz")); diff --git a/server/src/test/java/org/elasticsearch/common/settings/UpgradeSettingsIT.java b/server/src/test/java/org/elasticsearch/common/settings/UpgradeSettingsIT.java new file mode 100644 index 0000000000000..839b96e641870 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/settings/UpgradeSettingsIT.java @@ -0,0 +1,125 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.junit.After; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import static org.hamcrest.Matchers.equalTo; + +public class UpgradeSettingsIT extends ESSingleNodeTestCase { + + @After + public void cleanup() throws Exception { + client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("*")) + .setTransientSettings(Settings.builder().putNull("*")) + .get(); + } + + @Override + protected Collection> getPlugins() { + return Collections.singletonList(UpgradeSettingsPlugin.class); + } + + public static class UpgradeSettingsPlugin extends Plugin { + + static final Setting oldSetting = Setting.simpleString("foo.old", Setting.Property.Dynamic, Setting.Property.NodeScope); + static final Setting newSetting = Setting.simpleString("foo.new", Setting.Property.Dynamic, Setting.Property.NodeScope); + + public UpgradeSettingsPlugin(){ + + } + + @Override + public List> getSettings() { + return Arrays.asList(oldSetting, newSetting); + } + + @Override + public List> getSettingUpgraders() { + return Collections.singletonList(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + }); + } + } + + public void testUpgradePersistentSettingsOnUpdate() { + runUpgradeSettingsOnUpdateTest((settings, builder) -> builder.setPersistentSettings(settings), MetaData::persistentSettings); + } + + public void testUpgradeTransientSettingsOnUpdate() { + runUpgradeSettingsOnUpdateTest((settings, builder) -> builder.setTransientSettings(settings), MetaData::transientSettings); + } + + private void runUpgradeSettingsOnUpdateTest( + final BiConsumer consumer, + final Function settingsFunction) { + final String value = randomAlphaOfLength(8); + final ClusterUpdateSettingsRequestBuilder builder = + client() + .admin() + .cluster() + .prepareUpdateSettings(); + consumer.accept(Settings.builder().put("foo.old", value).build(), builder); + builder.get(); + + final ClusterStateResponse response = client() + .admin() + .cluster() + .prepareState() + .clear() + .setMetaData(true) + .get(); + + assertFalse(UpgradeSettingsPlugin.oldSetting.exists(settingsFunction.apply(response.getState().metaData()))); + assertTrue(UpgradeSettingsPlugin.newSetting.exists(settingsFunction.apply(response.getState().metaData()))); + assertThat(UpgradeSettingsPlugin.newSetting.get(settingsFunction.apply(response.getState().metaData())), equalTo("new." + value)); + } + +} diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayTests.java new file mode 100644 index 0000000000000..457b3a14ebf4a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayTests.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gateway; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.SettingUpgrader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; + +public class GatewayTests extends ESTestCase { + + public void testUpgradePersistentSettings() { + runUpgradeSettings(MetaData.Builder::persistentSettings, MetaData::persistentSettings); + } + + public void testUpgradeTransientSettings() { + runUpgradeSettings(MetaData.Builder::transientSettings, MetaData::transientSettings); + } + + private void runUpgradeSettings( + final BiConsumer applySettingsToBuilder, final Function metaDataSettings) { + final Setting oldSetting = Setting.simpleString("foo.old", Setting.Property.Dynamic, Setting.Property.NodeScope); + final Setting newSetting = Setting.simpleString("foo.new", Setting.Property.Dynamic, Setting.Property.NodeScope); + final Set> settingsSet = + Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of(oldSetting, newSetting)).collect(Collectors.toSet()); + final ClusterSettings clusterSettings = new ClusterSettings( + Settings.EMPTY, + settingsSet, + Collections.singleton(new SettingUpgrader() { + + @Override + public Setting getSetting() { + return oldSetting; + } + + @Override + public String getKey(final String key) { + return "foo.new"; + } + + @Override + public String getValue(final String value) { + return "new." + value; + } + + })); + final ClusterService clusterService = new ClusterService(Settings.EMPTY, clusterSettings, null); + final Gateway gateway = new Gateway(Settings.EMPTY, clusterService, null, null); + final MetaData.Builder builder = MetaData.builder(); + final Settings settings = Settings.builder().put("foo.old", randomAlphaOfLength(8)).build(); + applySettingsToBuilder.accept(builder, settings); + final ClusterState state = gateway.upgradeAndArchiveUnknownOrInvalidSettings(builder).build(); + assertFalse(oldSetting.exists(metaDataSettings.apply(state.metaData()))); + assertTrue(newSetting.exists(metaDataSettings.apply(state.metaData()))); + assertThat(newSetting.get(metaDataSettings.apply(state.metaData())), equalTo("new." + oldSetting.get(settings))); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index cc3902893411a..60f93f8ea30fe 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -340,7 +340,8 @@ private static class ServiceHolder implements Closeable { clientInvocationHandler); ScriptModule scriptModule = createScriptModule(pluginsService.filterPlugins(ScriptPlugin.class)); List> additionalSettings = pluginsService.getPluginSettings(); - SettingsModule settingsModule = new SettingsModule(nodeSettings, additionalSettings, pluginsService.getPluginSettingsFilter()); + SettingsModule settingsModule = + new SettingsModule(nodeSettings, additionalSettings, pluginsService.getPluginSettingsFilter(), Collections.emptySet()); searchModule = new SearchModule(nodeSettings, false, pluginsService.filterPlugins(SearchPlugin.class)); IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class)); List entries = new ArrayList<>(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java index f7fabab2799af..daf1775a80a52 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.core.security.SecurityExtension; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -181,7 +182,8 @@ private static void validateRealm(String name, String type, Settings settings, S settingSet.add(TYPE_SETTING); settingSet.add(ENABLED_SETTING); settingSet.add(ORDER_SETTING); - final AbstractScopedSettings validator = new AbstractScopedSettings(settings, settingSet, Setting.Property.NodeScope) { }; + final AbstractScopedSettings validator = + new AbstractScopedSettings(settings, settingSet, Collections.emptySet(), Setting.Property.NodeScope) { }; try { validator.validate(settings, false); } catch (RuntimeException e) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java index 1886dd4249b14..3bf3bb4dc8641 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java @@ -20,7 +20,9 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; + import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -121,7 +123,7 @@ public void testFiltering() throws Exception { List settingsFilterList = new ArrayList<>(); settingsFilterList.addAll(securityPlugin.getSettingsFilter()); // custom settings, potentially added by a plugin - SettingsModule settingsModule = new SettingsModule(settings, settingList, settingsFilterList); + SettingsModule settingsModule = new SettingsModule(settings, settingList, settingsFilterList, Collections.emptySet()); Injector injector = Guice.createInjector(settingsModule); SettingsFilter settingsFilter = injector.getInstance(SettingsFilter.class);