Introduce declarative plugin management (#77544)

Closes #70219.

Introduce a declarative way for the Elasticsearch server to manage plugins,
which reads the `elasticsearch-plugins.yml` file and works which out
plugins need to be added and / or removed to match the configuration. Also
make it possible to configure a proxy in the config file, instead of
through the environment.

Most of the work of adding and removing is still done in the
`InstallPluginAction` and `RemovePluginAction` classes, so the
behaviour should be the same as with the `install` and `remove`
commands. However, these commands will now abort if the above config
file exists. The intent is to make it harder for the configuration
to drift.

This new method only applies to `docker` distribution types at the
moment.

Since this syncing mechanism declarative, rather than imperative,
the Cloud-specific plugin wrapper script is no longer required.
Instead, an environment variable informs `InstallPluginAction` to
install plugins from an archive directory instead of downloading
them, where possible.

build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle

@@ -13,6 +13,7 @@
  * build/distributions/local
  * */
 import org.elasticsearch.gradle.Architecture
+import org.elasticsearch.gradle.VersionProperties
 
 // gradle has an open issue of failing applying plugins in
 // precompiled script plugins (see https://github.com/gradle/gradle/issues/17004)
@@ -29,6 +30,6 @@ tasks.register('localDistro', Sync) {
   from(elasticsearch_distributions.local)
   into("build/distribution/local")
   doLast {
-    logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}.")
+    logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}/elasticsearch-${VersionProperties.elasticsearch}")
   }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -102,7 +102,7 @@ public void apply(Project project) {
         Map<String, TaskProvider<?>> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest");
         TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
 
-        // Configuration examplePlugin = configureExamplePlugin(project);
+        Configuration examplePlugin = configureExamplePlugin(project);
 
         List<TaskProvider<Test>> windowsTestTasks = new ArrayList<>();
         Map<ElasticsearchDistributionType, List<TaskProvider<Test>>> linuxTestTasks = new HashMap<>();
@@ -114,11 +114,12 @@ public void apply(Project project) {
             TaskProvider<?> depsTask = project.getTasks().register(taskname + "#deps");
             // explicitly depend on the archive not on the implicit extracted distribution
             depsTask.configure(t -> t.dependsOn(distribution.getArchiveDependencies()));
+            depsTask.configure(t -> t.dependsOn(examplePlugin.getDependencies()));
             depsTasks.put(taskname, depsTask);
             TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
                 t.onlyIf(t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable);
                 addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
-                // addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
+                addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
                 t.exclude("**/PackageUpgradeTests.class");
             }, depsTask);
 
@@ -314,7 +315,7 @@ private static Configuration configureExamplePlugin(Project project) {
         Configuration examplePlugin = project.getConfigurations().create(EXAMPLE_PLUGIN_CONFIGURATION);
         examplePlugin.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
         DependencyHandler deps = project.getDependencies();
-        deps.add(EXAMPLE_PLUGIN_CONFIGURATION, deps.create("org.elasticsearch.examples:custom-settings:1.0.0-SNAPSHOT"));
+        deps.add(EXAMPLE_PLUGIN_CONFIGURATION, deps.project(Map.of("path", ":plugins:analysis-icu", "configuration", "zip")));
         return examplePlugin;
     }
 

build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml

@@ -34,7 +34,7 @@
         <message key="javadoc.missing" value="Types should explain their purpose" />
     </module>
 
-    <!-- Public methods must have JavaDoc -->
+    <!-- Check the Javadoc for a method e.g that it has the correct parameters, return type etc -->
     <module name="JavadocMethod">
         <property name="severity" value="warning"/>
         <property name="accessModifiers" value="public"/>

distribution/build.gradle

@@ -296,7 +296,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
      *****************************************************************************/
     libFiles =
       copySpec {
-        // delay by using closures, since they have not yet been configured, so no jar task exists yet
+        // Delay by using closures, since they have not yet been configured, so no jar task exists yet.
         from(configurations.libs)
         into('tools/geoip-cli') {
           from(configurations.libsGeoIpCli)

distribution/docker/build.gradle

@@ -265,12 +265,6 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         }
         // For some reason, the artifact name can differ depending on what repository we used.
         rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
-
-        into('bin') {
-          from(project.projectDir.toPath().resolve('src/docker/cloud')) {
-            expand([ version: VersionProperties.elasticsearch ])
-          }
-        }
       }
 
       onlyIf { Architecture.current() == architecture }

distribution/docker/src/docker/Dockerfile

@@ -262,12 +262,17 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
     find config -type f -exec chmod 0664 {} +
 
 <% if (docker_base == "cloud") { %>
-# Preinstall common plugins
+# Preinstall common plugins. Note that these are installed as root, meaning the `elasticsearch` user cannot delete them.
 COPY repository-s3-${version}.zip repository-gcs-${version}.zip repository-azure-${version}.zip /tmp/
-RUN bin/elasticsearch-plugin install --batch \\
+RUN bin/elasticsearch-plugin install --batch --verbose \\
     file:/tmp/repository-s3-${version}.zip \\
     file:/tmp/repository-gcs-${version}.zip \\
     file:/tmp/repository-azure-${version}.zip
+# Generate a replacement example plugins config that reflects what is actually installed
+RUN echo "plugins:" > config/elasticsearch-plugins.example.yml && \\
+    echo "  - id: repository-azure" >> config/elasticsearch-plugins.example.yml && \\
+    echo "  - id: repository-gcs"   >> config/elasticsearch-plugins.example.yml && \\
+    echo "  - id: repository-s3"    >> config/elasticsearch-plugins.example.yml
 
 <% /*  I tried to use `ADD` here, but I couldn't force it to do what I wanted */ %>
 COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
@@ -277,8 +282,6 @@ RUN mkdir -p /opt/filebeat /opt/metricbeat && \\
 
 # Add plugins infrastructure
 RUN mkdir -p /opt/plugins/archive
-COPY bin/plugin-wrapper.sh /opt/plugins
-# These are the correct permissions for both the directories and the script
 RUN chmod -R 0555 /opt/plugins
 <% } %>
 

distribution/docker/src/docker/Dockerfile.cloud-ess

@@ -10,3 +10,4 @@ RUN chmod 0444 /opt/plugins/archive/*
 FROM ${base_image}
 
 COPY --from=builder /opt/plugins /opt/plugins
+ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive

distribution/packages/build.gradle

@@ -175,6 +175,7 @@ def commonPackageConfig(String type, String architecture) {
 
     // ========= config files =========
     configurationFile '/etc/elasticsearch/elasticsearch.yml'
+    configurationFile '/etc/elasticsearch/elasticsearch-plugins.example.yml'
     configurationFile '/etc/elasticsearch/jvm.options'
     configurationFile '/etc/elasticsearch/log4j2.properties'
     configurationFile '/etc/elasticsearch/role_mapping.yml'

distribution/src/config/elasticsearch-plugins.example.yml

@@ -0,0 +1,27 @@
+# Rename this file to `elasticsearch-plugins.yml` to use it.
+#
+# All plugins must be listed here. If you add a plugin to this list and run
+# `elasticsearch-plugin sync`, that plugin will be installed. If you remove
+# a plugin from this list, that plugin will be removed when Elasticsearch
+# next starts.
+
+plugins:
+  # Each plugin must have an ID. Plugins with only an ID are official plugins and will be downloaded from Elastic.
+  - id: example-id
+
+  # Plugins can be specified by URL (it doesn't have to be HTTP, you could use e.g. `file:`)
+  - id: example-with-url
+    location: https://some.domain/path/example4.zip
+
+  # Or by maven coordinates:
+  - id: example-with-maven-url
+    location: org.elasticsearch.plugins:example-plugin:1.2.3
+
+  # A proxy can also be configured per-plugin, if necessary
+  - id: example-with-proxy
+    location: https://some.domain/path/example.zip
+    proxy: https://some.domain:1234
+
+# Configures a proxy for all network access. Remove this if you don't need
+# to use a proxy.
+proxy: https://some.domain:1234

distribution/tools/plugin-cli/build.gradle

@@ -13,10 +13,6 @@ archivesBaseName = 'elasticsearch-plugin-cli'
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:elasticsearch-cli")
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
   api "org.bouncycastle:bcpg-fips:1.0.4"
   api "org.bouncycastle:bc-fips:1.0.2"
   testImplementation project(":test:framework")