Introduce licensed plugins (#65490)

Backport of #64850.

This PR introduces the concept of "licensed" plugins. Such plugins
may only be installed on installations of the default distribution,
and this is enforced by the plugin installer. This PR also moves
the `quote-aware-fs` plugin to the `x-pack` directory, and marks
it as licensed.

Note that I didn't move the plugin source under `x-pack/plugin`
because all the existing x-pack plugins are actually bundles as
modules into the default distribution, whereas the `quota-aware-fs`
plugin needs to remain a standalone plugin.

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -46,7 +46,7 @@ public class PluginBuildPlugin extends BuildPlugin {
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
         project.afterEvaluate {
-            boolean isXPackModule = project.path.startsWith(':x-pack:plugin')
+            boolean isXPackModule = project.path.startsWith(':x-pack:plugin') || project.path.startsWith(':x-pack:quota-aware-fs')
             boolean isModule = project.path.startsWith(':modules:') || isXPackModule
             String name = project.pluginProperties.extension.name
             project.archivesBaseName = name

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -64,6 +64,10 @@ class PluginPropertiesExtension {
      */
     private File noticeFile = null
 
+    /** True if the plugin is available under the Elastic license. */
+    @Input
+    boolean licensed = false
+
     Project project = null
 
     PluginPropertiesExtension(Project project) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -81,7 +81,8 @@ class PluginPropertiesTask extends Copy {
             'classname': extension.classname,
             'extendedPlugins': extension.extendedPlugins.join(','),
             'hasNativeController': extension.hasNativeController,
-            'requiresKeystore': extension.requiresKeystore
+            'requiresKeystore': extension.requiresKeystore,
+            'licensed': extension.licensed
         ]
     }
 }

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -43,3 +43,7 @@ extended.plugins=${extendedPlugins}
 #
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
+<% if (licensed) { %>
+# This plugin requires that a license agreement be accepted before installation
+licensed=${licensed}
+<% } %>

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -801,6 +801,9 @@ private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environme
     private void installPlugin(Terminal terminal, boolean isBatch, Path tmpRoot,
                                Environment env, List<Path> deleteOnFailure) throws Exception {
         final PluginInfo info = loadPluginInfo(terminal, tmpRoot, env);
+
+        checkCanInstallationProceed(terminal, Build.CURRENT.flavor(), info);
+
         // read optional security policy (extra permissions), if it exists, confirm or warn the user
         Path policy = tmpRoot.resolve(PluginInfo.ES_PLUGIN_POLICY);
         final Set<String> permissions;
@@ -950,4 +953,24 @@ public void close() throws IOException {
         IOUtils.rm(pathsToDeleteOnShutdown.toArray(new Path[pathsToDeleteOnShutdown.size()]));
     }
 
+    static void checkCanInstallationProceed(Terminal terminal, Build.Flavor flavor, PluginInfo info) throws Exception {
+        if (info.isLicensed() == false) {
+            return;
+        }
+
+        if (flavor == Build.Flavor.DEFAULT) {
+            return;
+        }
+
+        Arrays.asList(
+            "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@",
+            "@            ERROR: This is a licensed plugin             @",
+            "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@",
+            "",
+            "This plugin is covered by the Elastic license, but this",
+            "installation of Elasticsearch is: [" + flavor + "]."
+        ).forEach(terminal::println);
+
+        throw new UserException(ExitCodes.NOPERM, "Plugin license is incompatible with [" + flavor + "] installation");
+    }
 }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallLicensedPluginTests.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+import java.util.Collections;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.test.ESTestCase;
+
+public class InstallLicensedPluginTests extends ESTestCase {
+
+    /**
+     * Check that an unlicensed plugin is accepted.
+     */
+    public void testUnlicensedPlugin() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        PluginInfo pluginInfo = buildInfo(false);
+        InstallPluginCommand.checkCanInstallationProceed(terminal, Build.Flavor.OSS, pluginInfo);
+    }
+
+    /**
+     * Check that a licensed plugin cannot be installed on OSS.
+     */
+    public void testInstallPluginCommandOnOss() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        PluginInfo pluginInfo = buildInfo(true);
+        final UserException userException = expectThrows(
+            UserException.class,
+            () -> InstallPluginCommand.checkCanInstallationProceed(terminal, Build.Flavor.OSS, pluginInfo)
+        );
+
+        assertThat(userException.exitCode, equalTo(ExitCodes.NOPERM));
+        assertThat(terminal.getOutput(), containsString("ERROR: This is a licensed plugin"));
+    }
+
+    /**
+     * Check that a licensed plugin cannot be installed when the distribution type is unknown.
+     */
+    public void testInstallPluginCommandOnUnknownDistribution() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        PluginInfo pluginInfo = buildInfo(true);
+        expectThrows(
+            UserException.class,
+            () -> InstallPluginCommand.checkCanInstallationProceed(terminal, Build.Flavor.UNKNOWN, pluginInfo)
+        );
+        assertThat(terminal.getOutput(), containsString("ERROR: This is a licensed plugin"));
+    }
+
+    /**
+     * Check that a licensed plugin can be installed when the distribution type is default.
+     */
+    public void testInstallPluginCommandOnDefault() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        PluginInfo pluginInfo = buildInfo(true);
+        InstallPluginCommand.checkCanInstallationProceed(terminal, Build.Flavor.DEFAULT, pluginInfo);
+    }
+
+    private PluginInfo buildInfo(boolean isLicensed) {
+        return new PluginInfo(
+            "name",
+            "description",
+            "version",
+            Version.CURRENT,
+            "java version",
+            "classname",
+            Collections.emptyList(),
+            false,
+            isLicensed
+        );
+    }
+}

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -829,14 +829,14 @@ public void testPluginAlreadyInstalled() throws Exception {
                 "if you need to update the plugin, uninstall it first using command 'remove fake'"));
     }
 
-    private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
+    private void installPlugin(MockTerminal terminal, boolean isBatch, String... additionalProperties) throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         // if batch is enabled, we also want to add a security policy
         if (isBatch) {
             writePluginSecurityPolicy(pluginDir, "setFactory");
         }
-        String pluginZip = createPlugin("fake", pluginDir).toUri().toURL().toString();
+        String pluginZip = createPlugin("fake", pluginDir, additionalProperties).toUri().toURL().toString();
         skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2());
     }
 

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java

@@ -152,6 +152,7 @@ public void testPluginWithVerbose() throws Exception {
                         "Elasticsearch Version: " + Version.CURRENT.toString(),
                         "Java Version: 1.8",
                         "Native Controller: false",
+                        "Licensed: false",
                         "Extended Plugins: []",
                         " * Classname: org.fake"),
                 terminal.getOutput());
@@ -172,6 +173,7 @@ public void testPluginWithNativeController() throws Exception {
                 "Elasticsearch Version: " + Version.CURRENT.toString(),
                 "Java Version: 1.8",
                 "Native Controller: true",
+                "Licensed: false",
                 "Extended Plugins: []",
                 " * Classname: org.fake"),
             terminal.getOutput());
@@ -193,6 +195,7 @@ public void testPluginWithVerboseMultiplePlugins() throws Exception {
                         "Elasticsearch Version: " + Version.CURRENT.toString(),
                         "Java Version: 1.8",
                         "Native Controller: false",
+                        "Licensed: false",
                         "Extended Plugins: []",
                         " * Classname: org.fake",
                         "fake_plugin2",
@@ -203,6 +206,7 @@ public void testPluginWithVerboseMultiplePlugins() throws Exception {
                         "Elasticsearch Version: " + Version.CURRENT.toString(),
                         "Java Version: 1.8",
                         "Native Controller: false",
+                        "Licensed: false",
                         "Extended Plugins: []",
                         " * Classname: org.fake2"),
                 terminal.getOutput());

docs/build.gradle

@@ -71,9 +71,8 @@ integTestCluster {
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
   /* Skip repositories. We just aren't going to be able to test them so it
    * doesn't make sense to waste time installing them.
-   * Also skip quota-aware-fs since it has to be configured in order to use
-   * it, otherwise ES will not start. */
-  if (subproj.path.startsWith(':plugins:repository-') || subproj.path.startsWith(':plugins:quota-aware-fs')) {
+   */
+  if (subproj.path.startsWith(':plugins:repository-')) {
     return
   }
   subproj.afterEvaluate { // need to wait until the project has been configured

qa/smoke-test-plugins/build.gradle

@@ -24,10 +24,6 @@ apply plugin: 'elasticsearch.rest-test'
 
 ext.pluginsCount = 0
 project(':plugins').getChildProjects().each { pluginName, pluginProject ->
-  if (pluginName == 'quota-aware-fs') {
-    // This plugin has to be configured to work via system properties
-    return
-  }
   integTestCluster {
     plugin pluginProject.path
   }