Add infrastructure to upgrade settings (#33536)

In some cases we want to deprecate a setting, and then automatically
upgrade uses of that setting to a replacement setting. This commit adds
infrastructure for this so that we can upgrade settings when recovering
the cluster state, as well as when such settings are dynamically applied
on cluster update settings requests. This commit only focuses on cluster
settings, index settings can build on this infrastructure in a
follow-up.

server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java

@@ -179,8 +179,12 @@ public void onFailure(String source, Exception e) {
 
             @Override
             public ClusterState execute(final ClusterState currentState) {
-                ClusterState clusterState =
-                        updater.updateSettings(currentState, request.transientSettings(), request.persistentSettings(), logger);
+                final ClusterState clusterState =
+                        updater.updateSettings(
+                                currentState,
+                                clusterSettings.upgradeSettings(request.transientSettings()),
+                                clusterSettings.upgradeSettings(request.persistentSettings()),
+                                logger);
                 changed = clusterState != currentState;
                 return clusterState;
             }

server/src/main/java/org/elasticsearch/client/transport/TransportClient.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.client.transport;
 
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionModule;
@@ -44,6 +43,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.node.InternalSettingsPreparer;
@@ -146,7 +146,8 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
             for (final ExecutorBuilder<?> builder : threadPool.builders()) {
                 additionalSettings.addAll(builder.getRegisteredSettings());
             }
-            SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
+            SettingsModule settingsModule =
+                    new SettingsModule(settings, additionalSettings, additionalSettingsFilter, Collections.emptySet());
 
             SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class));
             IndicesModule indicesModule = new IndicesModule(Collections.emptyList());

server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
 
+import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -38,6 +39,7 @@
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -52,14 +54,29 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
     private final List<SettingUpdater<?>> settingUpdaters = new CopyOnWriteArrayList<>();
     private final Map<String, Setting<?>> complexMatchers;
     private final Map<String, Setting<?>> keySettings;
+    private final Map<Setting<?>, Function<Map.Entry<String, String>, Map.Entry<String, String>>> settingUpgraders;
     private final Setting.Property scope;
     private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$");
     private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$");
     private static final Pattern AFFIX_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+[*](?:[.][-\\w]+)+$");
 
-    protected AbstractScopedSettings(Settings settings, Set<Setting<?>> settingsSet, Setting.Property scope) {
+    protected AbstractScopedSettings(
+            final Settings settings,
+            final Set<Setting<?>> settingsSet,
+            final Set<SettingUpgrader<?>> settingUpgraders,
+            final Setting.Property scope) {
         super(settings);
         this.lastSettingsApplied = Settings.EMPTY;
+
+        this.settingUpgraders =
+                Collections.unmodifiableMap(
+                        settingUpgraders
+                                .stream()
+                                .collect(
+                                        Collectors.toMap(
+                                                SettingUpgrader::getSetting,
+                                                u -> e -> new AbstractMap.SimpleEntry<>(u.getKey(e.getKey()), u.getValue(e.getValue())))));
+
         this.scope = scope;
         Map<String, Setting<?>> complexMatchers = new HashMap<>();
         Map<String, Setting<?>> keySettings = new HashMap<>();
@@ -97,6 +114,7 @@ protected AbstractScopedSettings(Settings nodeSettings, Settings scopeSettings,
         this.scope = other.scope;
         complexMatchers = other.complexMatchers;
         keySettings = other.keySettings;
+        settingUpgraders = Collections.unmodifiableMap(new HashMap<>(other.settingUpgraders));
         settingUpdaters.addAll(other.settingUpdaters);
     }
 
@@ -757,6 +775,32 @@ private static Setting<?> findOverlappingSetting(Setting<?> newSetting, Map<Stri
         return null;
     }
 
+    /**
+     * Upgrade all settings eligible for upgrade in the specified settings instance.
+     *
+     * @param settings the settings instance that might contain settings to be upgraded
+     * @return a new settings instance if any settings required upgrade, otherwise the same settings instance as specified
+     */
+    public Settings upgradeSettings(final Settings settings) {
+        final Settings.Builder builder = Settings.builder();
+        boolean changed = false; // track if any settings were upgraded
+        for (final String key : settings.keySet()) {
+            final Setting<?> setting = getRaw(key);
+            final Function<Map.Entry<String, String>, Map.Entry<String, String>> upgrader = settingUpgraders.get(setting);
+            if (upgrader == null) {
+                // the setting does not have an upgrader, copy the setting
+                builder.copy(key, settings);
+            } else {
+                // the setting has an upgrader, so mark that we have changed a setting and apply the upgrade logic
+                changed = true;
+                final Map.Entry<String, String> upgrade = upgrader.apply(new Entry(key, settings));
+                builder.put(upgrade.getKey(), upgrade.getValue());
+            }
+        }
+        // we only return a new instance if there was an upgrade
+        return changed ? builder.build() : settings;
+    }
+
     /**
      * Archives invalid or unknown settings. Any setting that is not recognized or fails validation
      * will be archived. This means the setting is prefixed with {@value ARCHIVED_SETTINGS_PREFIX}
@@ -847,4 +891,5 @@ public String setValue(String value) {
     public boolean isPrivateSetting(String key) {
         return false;
     }
+
 }

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -100,15 +100,21 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
 
 /**
  * Encapsulates all valid cluster level settings.
  */
 public final class ClusterSettings extends AbstractScopedSettings {
-    public ClusterSettings(Settings nodeSettings, Set<Setting<?>> settingsSet) {
-        super(nodeSettings, settingsSet, Property.NodeScope);
+    public ClusterSettings(final Settings nodeSettings, final Set<Setting<?>> settingsSet) {
+        this(nodeSettings, settingsSet, Collections.emptySet());
+    }
+
+    public ClusterSettings(
+            final Settings nodeSettings, final Set<Setting<?>> settingsSet, final Set<SettingUpgrader<?>> settingUpgraders) {
+        super(nodeSettings, settingsSet, settingUpgraders, Property.NodeScope);
         addSettingsUpdater(new LoggingSettingUpdater(nodeSettings));
     }
 
@@ -436,4 +442,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     IndexGraveyard.SETTING_MAX_TOMBSTONES,
                     EnableAssignmentDecider.CLUSTER_TASKS_ALLOCATION_ENABLE_SETTING
             )));
+
+    public static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
+
 }

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -178,7 +178,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
     public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);
 
     public IndexScopedSettings(Settings settings, Set<Setting<?>> settingsSet) {
-        super(settings, settingsSet, Property.IndexScope);
+        super(settings, settingsSet, Collections.emptySet(), Property.IndexScope);
     }
 
     private IndexScopedSettings(Settings settings, IndexScopedSettings other, IndexMetaData metaData) {

server/src/main/java/org/elasticsearch/common/settings/SettingUpgrader.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+/**
+ * Represents the logic to upgrade a setting.
+ *
+ * @param <T> the type of the underlying setting
+ */
+public interface SettingUpgrader<T> {
+
+    /**
+     * The setting upgraded by this upgrader.
+     *
+     * @return the setting
+     */
+    Setting<T> getSetting();
+
+    /**
+     * The logic to upgrade the setting key, for example by mapping the old setting key to the new setting key.
+     *
+     * @param key the setting key to upgrade
+     * @return the upgraded setting key
+     */
+    String getKey(String key);
+
+    /**
+     * The logic to upgrade the setting value.
+     *
+     * @param value the setting value to upgrade
+     * @return the upgraded setting value
+     */
+    default String getValue(final String value) {
+        return value;
+    }
+
+}

server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java

@@ -54,10 +54,14 @@ public class SettingsModule implements Module {
     private final SettingsFilter settingsFilter;
 
     public SettingsModule(Settings settings, Setting<?>... additionalSettings) {
-        this(settings, Arrays.asList(additionalSettings), Collections.emptyList());
+        this(settings, Arrays.asList(additionalSettings), Collections.emptyList(), Collections.emptySet());
     }
 
-    public SettingsModule(Settings settings, List<Setting<?>> additionalSettings, List<String> settingsFilter) {
+    public SettingsModule(
+            Settings settings,
+            List<Setting<?>> additionalSettings,
+            List<String> settingsFilter,
+            Set<SettingUpgrader<?>> settingUpgraders) {
         logger = Loggers.getLogger(getClass(), settings);
         this.settings = settings;
         for (Setting<?> setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) {
@@ -70,12 +74,22 @@ public SettingsModule(Settings settings, List<Setting<?>> additionalSettings, Li
         for (Setting<?> setting : additionalSettings) {
             registerSetting(setting);
         }
-
         for (String filter : settingsFilter) {
             registerSettingsFilter(filter);
         }
+        final Set<SettingUpgrader<?>> clusterSettingUpgraders = new HashSet<>();
+        for (final SettingUpgrader<?> settingUpgrader : ClusterSettings.BUILT_IN_SETTING_UPGRADERS) {
+            assert settingUpgrader.getSetting().hasNodeScope() : settingUpgrader.getSetting().getKey();
+            final boolean added = clusterSettingUpgraders.add(settingUpgrader);
+            assert added : settingUpgrader.getSetting().getKey();
+        }
+        for (final SettingUpgrader<?> settingUpgrader : settingUpgraders) {
+            assert settingUpgrader.getSetting().hasNodeScope() : settingUpgrader.getSetting().getKey();
+            final boolean added = clusterSettingUpgraders.add(settingUpgrader);
+            assert added : settingUpgrader.getSetting().getKey();
+        }
         this.indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values()));
-        this.clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()));
+        this.clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()), clusterSettingUpgraders);
         Settings indexSettings = settings.filter((s) -> (s.startsWith("index.") &&
             // special case - we want to get Did you mean indices.query.bool.max_clause_count
             // which means we need to by-pass this check for this setting
@@ -205,4 +219,5 @@ public ClusterSettings getClusterSettings() {
     public SettingsFilter getSettingsFilter() {
         return settingsFilter;
     }
+
 }

server/src/main/java/org/elasticsearch/gateway/Gateway.java

@@ -137,20 +137,25 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t
                 }
             }
         }
+        final ClusterState.Builder builder = upgradeAndArchiveUnknownOrInvalidSettings(metaDataBuilder);
+        listener.onSuccess(builder.build());
+    }
+
+    ClusterState.Builder upgradeAndArchiveUnknownOrInvalidSettings(MetaData.Builder metaDataBuilder) {
         final ClusterSettings clusterSettings = clusterService.getClusterSettings();
         metaDataBuilder.persistentSettings(
             clusterSettings.archiveUnknownOrInvalidSettings(
-                metaDataBuilder.persistentSettings(),
+                clusterSettings.upgradeSettings(metaDataBuilder.persistentSettings()),
                 e -> logUnknownSetting("persistent", e),
                 (e, ex) -> logInvalidSetting("persistent", e, ex)));
         metaDataBuilder.transientSettings(
             clusterSettings.archiveUnknownOrInvalidSettings(
-                metaDataBuilder.transientSettings(),
+                clusterSettings.upgradeSettings(metaDataBuilder.transientSettings()),
                 e -> logUnknownSetting("transient", e),
                 (e, ex) -> logInvalidSetting("transient", e, ex)));
         ClusterState.Builder builder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings));
         builder.metaData(metaDataBuilder);
-        listener.onSuccess(builder.build());
+        return builder;
     }
 
     private void logUnknownSetting(String settingType, Map.Entry<String, String> e) {

server/src/main/java/org/elasticsearch/node/Node.java

@@ -73,6 +73,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.SettingUpgrader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.BoundTransportAddress;
@@ -151,6 +152,7 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import javax.net.ssl.SNIHostName;
+
 import java.io.BufferedWriter;
 import java.io.Closeable;
 import java.io.IOException;
@@ -360,7 +362,15 @@ protected Node(
             AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class));
             // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
             // so we might be late here already
-            final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter);
+
+            final Set<SettingUpgrader<?>> settingsUpgraders = pluginsService.filterPlugins(Plugin.class)
+                    .stream()
+                    .map(Plugin::getSettingUpgraders)
+                    .flatMap(List::stream)
+                    .collect(Collectors.toSet());
+
+            final SettingsModule settingsModule =
+                    new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter, settingsUpgraders);
             scriptModule.registerClusterSettingsListeners(settingsModule.getClusterSettings());
             resourcesToClose.add(resourceWatcherService);
             final NetworkService networkService = new NetworkService(

server/src/main/java/org/elasticsearch/plugins/Plugin.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.SettingUpgrader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -172,6 +173,15 @@ public void onIndexModule(IndexModule indexModule) {}
      */
     public List<String> getSettingsFilter() { return Collections.emptyList(); }
 
+    /**
+     * Get the setting upgraders provided by this plugin.
+     *
+     * @return the settings upgraders
+     */
+    public List<SettingUpgrader<?>> getSettingUpgraders() {
+        return Collections.emptyList();
+    }
+
     /**
      * Provides a function to modify global custom meta data on startup.
      * <p>