Allow plugins to upgrade templates and index metadata on startup

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java

@@ -35,8 +35,10 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.UpgraderPlugin;
 
 import java.util.AbstractMap;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
@@ -54,14 +56,16 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
     private final NamedXContentRegistry xContentRegistry;
     private final MapperRegistry mapperRegistry;
     private final IndexScopedSettings indexScopedSettings;
+    private final Collection<UpgraderPlugin> upgraderPlugins;
 
     @Inject
     public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
-            IndexScopedSettings indexScopedSettings) {
+                                       IndexScopedSettings indexScopedSettings, Collection<UpgraderPlugin> upgraderPlugins) {
         super(settings);
         this.xContentRegistry = xContentRegistry;
         this.mapperRegistry = mapperRegistry;
         this.indexScopedSettings = indexScopedSettings;
+        this.upgraderPlugins = upgraderPlugins;
     }
 
     /**
@@ -84,6 +88,10 @@ public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version m
         newMetaData = archiveBrokenIndexSettings(newMetaData);
         // only run the check with the upgraded settings!!
         checkMappingsCompatibility(newMetaData);
+        // apply plugin checks
+        for (UpgraderPlugin upgraderPlugin : upgraderPlugins) {
+            newMetaData = upgraderPlugin.getIndexMetaDataUpgrader().apply(newMetaData);
+        }
         return markAsUpgraded(newMetaData);
     }
 

core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -50,6 +51,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
 
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableSet;
@@ -219,8 +223,8 @@ private void ensureNoPre019State() throws Exception {
                     final String name = stateFile.getFileName().toString();
                     if (name.startsWith("metadata-")) {
                         throw new IllegalStateException("Detected pre 0.19 metadata file please upgrade to a version before "
-                                + Version.CURRENT.minimumCompatibilityVersion()
-                                + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath());
+                            + Version.CURRENT.minimumCompatibilityVersion()
+                            + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath());
                     }
                 }
             }
@@ -247,23 +251,41 @@ static MetaData upgradeMetaData(MetaData metaData,
             changed |= indexMetaData != newMetaData;
             upgradedMetaData.put(newMetaData, false);
         }
-        // collect current customs
-        Map<String, MetaData.Custom> existingCustoms = new HashMap<>();
-        for (ObjectObjectCursor<String, MetaData.Custom> customCursor : metaData.customs()) {
-            existingCustoms.put(customCursor.key, customCursor.value);
-        }
         // upgrade global custom meta data
-        Map<String, MetaData.Custom> upgradedCustoms = metaDataUpgrader.customMetaDataUpgraders.apply(existingCustoms);
-        if (upgradedCustoms.equals(existingCustoms) == false) {
-            existingCustoms.keySet().forEach(upgradedMetaData::removeCustom);
-            for (Map.Entry<String, MetaData.Custom> upgradedCustomEntry : upgradedCustoms.entrySet()) {
-                upgradedMetaData.putCustom(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
-            }
+        if (applyPluginUpraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
+            upgradedMetaData::removeCustom,upgradedMetaData::putCustom)) {
+            changed = true;
+        }
+        // upgrade current templates
+        if (applyPluginUpraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
+            upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
             changed = true;
         }
         return changed ? upgradedMetaData.build() : metaData;
     }
 
+    private static <Data> boolean applyPluginUpraders(ImmutableOpenMap<String, Data> existingData,
+                                               UnaryOperator<Map<String, Data>> upgrader,
+                                               Consumer<String> removeData,
+                                               BiConsumer<String, Data> putData) {
+        // collect current data
+        Map<String, Data> existingMap = new HashMap<>();
+        for (ObjectObjectCursor<String, Data> customCursor : existingData) {
+            existingMap.put(customCursor.key, customCursor.value);
+        }
+        // upgrade global custom meta data
+        Map<String, Data> upgradedCustoms = upgrader.apply(existingMap);
+        if (upgradedCustoms.equals(existingMap) == false) {
+            // remove all data first so a plugin can remove custom metadata or templates if needed
+            existingMap.keySet().forEach(removeData);
+            for (Map.Entry<String, Data> upgradedCustomEntry : upgradedCustoms.entrySet()) {
+                putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
+            }
+            return true;
+        }
+        return false;
+    }
+
     // shard state BWC
     private void ensureNoPre019ShardState(NodeEnvironment nodeEnv) throws Exception {
         for (Path dataLocation : nodeEnv.nodeDataPaths()) {

core/src/main/java/org/elasticsearch/node/Node.java

@@ -115,6 +115,7 @@
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.plugins.UpgraderPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.script.ScriptModule;
@@ -402,14 +403,11 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                 .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
                                                  scriptModule.getScriptService(), xContentRegistry).stream())
                 .collect(Collectors.toList());
-            Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
-                pluginsService.filterPlugins(Plugin.class).stream()
-                .map(Plugin::getCustomMetaDataUpgrader)
-                .collect(Collectors.toList());
+            Collection<UpgraderPlugin> upgraderPlugins = pluginsService.filterPlugins(UpgraderPlugin.class);
             final RestController restController = actionModule.getRestController();
             final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
                     threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
-            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders);
+            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(upgraderPlugins);
             final Transport transport = networkModule.getTransportSupplier().get();
             final TransportService transportService = newTransportService(settings, transport, threadPool,
                 networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings());
@@ -464,7 +462,7 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                     b.bind(NetworkService.class).toInstance(networkService);
                     b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService()));
                     b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings,
-                        xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings()));
+                        xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), upgraderPlugins));
                     b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
                     b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery());
                     {

core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 
 import java.util.Collection;
@@ -32,13 +33,23 @@
 public class MetaDataUpgrader {
     public final UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgraders;
 
-    public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders) {
+    public final UnaryOperator<Map<String, IndexTemplateMetaData>> indexTemplateMetaDataUpgraders;
+
+    public MetaDataUpgrader(Collection<UpgraderPlugin> upgraderPlugins) {
         this.customMetaDataUpgraders = customs -> {
             Map<String, MetaData.Custom> upgradedCustoms = new HashMap<>(customs);
-            for (UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgrader : customMetaDataUpgraders) {
-                upgradedCustoms = customMetaDataUpgrader.apply(upgradedCustoms);
+            for (UpgraderPlugin upgraderPlugin : upgraderPlugins) {
+                upgradedCustoms = upgraderPlugin.getCustomMetaDataUpgrader().apply(upgradedCustoms);
             }
             return upgradedCustoms;
         };
+
+        this.indexTemplateMetaDataUpgraders = templates -> {
+            Map<String, IndexTemplateMetaData> upgradedTemplates = new HashMap<>(templates);
+            for (UpgraderPlugin upgraderPlugin : upgraderPlugins) {
+                upgradedTemplates = upgraderPlugin.getIndexTemplateMetaDataUpgrader().apply(upgradedTemplates);
+            }
+            return upgradedTemplates;
+        };
     }
 }

core/src/main/java/org/elasticsearch/plugins/Plugin.java

@@ -23,7 +23,6 @@
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterModule;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
@@ -51,8 +50,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.function.UnaryOperator;
 
 /**
  * An extension point allowing to plug in custom functionality. This class has a number of extension points that are available to all
@@ -68,6 +65,7 @@
  * <li>{@link RepositoryPlugin}
  * <li>{@link ScriptPlugin}
  * <li>{@link SearchPlugin}
+ * <li>{@link UpgraderPlugin}
  * </ul>
  * <p>In addition to extension points this class also declares some {@code @Deprecated} {@code public final void onModule} methods. These
  * methods should cause any extensions of {@linkplain Plugin} that used the pre-5.x style extension syntax to fail to build and point the
@@ -149,18 +147,6 @@ public void onIndexModule(IndexModule indexModule) {}
      */
     public List<String> getSettingsFilter() { return Collections.emptyList(); }
 
-    /**
-     * Provides a function to modify global custom meta data on startup.
-     * <p>
-     * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required.
-     * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map.
-     * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom}
-     *         is unsupported
-     */
-    public UnaryOperator<Map<String, MetaData.Custom>> getCustomMetaDataUpgrader() {
-        return UnaryOperator.identity();
-    }
-
     /**
      * Provides the list of this plugin's custom thread pools, empty if
      * none.

core/src/main/java/org/elasticsearch/plugins/UpgraderPlugin.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+
+import java.util.Map;
+import java.util.function.UnaryOperator;
+
+/**
+ * Extension points for upgrading cluster and index metadata
+ */
+public class UpgraderPlugin {
+    /**
+     * Provides a function to modify global custom meta data on startup.
+     * <p>
+     * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map.
+     * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom}
+     *         is unsupported
+     */
+    public UnaryOperator<Map<String, MetaData.Custom>> getCustomMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
+    /**
+     * Provides a function to modify index template meta data on startup.
+     * <p>
+     * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map.
+     * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData}
+     *         cannot be upgraded
+     */
+    public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
+    /**
+     * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time.
+     * <p>
+     * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required.
+     * @return Never {@code null}. The same or upgraded {@code IndexMetaData}.
+     * @throws IllegalStateException if the node should not start because the index is unsupported
+     */
+    public UnaryOperator<IndexMetaData> getIndexMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
+
+}