Verify signatures on official plugins

distribution/build.gradle

@@ -227,13 +227,16 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
      *                   Common files in all distributions                       *
      *****************************************************************************/
     libFiles = copySpec {
-      into 'lib'
-      from { project(':server').jar }
-      from { project(':server').configurations.runtime }
-      from { project(':libs:plugin-classloader').jar }
-      // delay add tools using closures, since they have not yet been configured, so no jar task exists yet
-      from { project(':distribution:tools:launchers').jar }
-      from { project(':distribution:tools:plugin-cli').jar }
+      into('lib') {
+        from { project(':server').jar }
+        from { project(':server').configurations.runtime }
+        from { project(':libs:plugin-classloader').jar }
+        from { project(':distribution:tools:launchers').jar }
+        into('tools/plugin-cli') {
+          from { project(':distribution:tools:plugin-cli').jar }
+          from { project(':distribution:tools:plugin-cli').configurations.runtime }
+        }
+      }
     }
 
     modulesFiles = { oss ->

distribution/src/bin/elasticsearch-cli

@@ -10,6 +10,12 @@ do
   source "`dirname "$0"`"/$additional_source
 done
 
+IFS=';' read -r -a additional_classpath_directories <<< "$ES_ADDITIONAL_CLASSPATH_DIRECTORIES"
+for additional_classpath_directory in "${additional_classpath_directories[@]}"
+do
+  ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/$additional_classpath_directory/*"
+done
+
 exec \
   "$JAVA" \
   $ES_JAVA_OPTS \

distribution/src/bin/elasticsearch-cli.bat

@@ -11,6 +11,12 @@ for /f "tokens=1*" %%a in ("%*") do (
   set arguments=%%b
 )
 
+if defined ES_ADDITIONAL_CLASSPATH_DIRECTORIES (
+  for %%a in ("%ES_ADDITIONAL_CLASSPATH_DIRECTORIES:;=","%") do (
+    ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/%%a/*
+  )
+)
+
 %JAVA% ^
   %ES_JAVA_OPTS% ^
   -Des.path.home="%ES_HOME%" ^

distribution/src/bin/elasticsearch-plugin

@@ -1,5 +1,6 @@
 #!/bin/bash
 
-"`dirname "$0"`"/elasticsearch-cli \
+ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/plugin-cli \
+  "`dirname "$0"`"/elasticsearch-cli \
   org.elasticsearch.plugins.PluginCli \
   "$@"

distribution/src/bin/elasticsearch-plugin.bat

@@ -3,6 +3,7 @@
 setlocal enabledelayedexpansion
 setlocal enableextensions
 
+set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/plugin-cli
 call "%~dp0elasticsearch-cli.bat" ^
   org.elasticsearch.plugins.PluginCli ^
   %* ^

distribution/tools/plugin-cli/build.gradle

@@ -19,14 +19,22 @@
 
 apply plugin: 'elasticsearch.build'
 
+archivesBaseName = 'elasticsearch-plugin-cli'
+
 dependencies {
   compileOnly "org.elasticsearch:elasticsearch:${version}"
   compileOnly "org.elasticsearch:elasticsearch-cli:${version}"
+  compile "org.bouncycastle:bcpg-jdk15on:1.59"
+  compile "org.bouncycastle:bcprov-jdk15on:1.59"
   testCompile "org.elasticsearch.test:framework:${version}"
   testCompile 'com.google.jimfs:jimfs:1.1'
   testCompile 'com.google.guava:guava:18.0'
 }
 
+dependencyLicenses {
+  mapping from: /bc.*/, to: 'bouncycastle'
+}
+
 test {
   // TODO: find a way to add permissions for the tests in this module
   systemProperty 'tests.security.manager', 'false'

distribution/tools/plugin-cli/licenses/bcpg-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+ee93e5376bb6cf0a15c027b5f5e4393f2738e709

distribution/tools/plugin-cli/licenses/bcprov-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+2507204241ab450456bdb8e8c0a8f986e418bd99

distribution/tools/plugin-cli/licenses/bouncycastle-LICENSE.txt

@@ -0,0 +1,17 @@
+Copyright (c) 2000-2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software 
+and associated documentation files (the "Software"), to deal in the Software without restriction, 
+including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

distribution/tools/plugin-cli/licenses/bouncycastle-NOTICE.txt

@@ -0,0 +1 @@
+

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -23,6 +23,16 @@
 import joptsimple.OptionSpec;
 import org.apache.lucene.search.spell.LevensteinDistance;
 import org.apache.lucene.util.CollectionUtil;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.openpgp.PGPException;
+import org.bouncycastle.openpgp.PGPPublicKey;
+import org.bouncycastle.openpgp.PGPPublicKeyRingCollection;
+import org.bouncycastle.openpgp.PGPSignature;
+import org.bouncycastle.openpgp.PGPSignatureList;
+import org.bouncycastle.openpgp.PGPUtil;
+import org.bouncycastle.openpgp.jcajce.JcaPGPObjectFactory;
+import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator;
+import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider;
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
@@ -43,6 +53,7 @@
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLDecoder;
@@ -59,8 +70,11 @@
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -116,7 +130,6 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
     /** The plugin zip is not properly structured. */
     static final int PLUGIN_MALFORMED = 2;
 
-
     /** The builtin modules, which are plugins, but cannot be installed or removed. */
     static final Set<String> MODULES;
     static {
@@ -241,15 +254,15 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Ex
         if (OFFICIAL_PLUGINS.contains(pluginId)) {
             final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME);
             terminal.println("-> Downloading " + pluginId + " from elastic");
-            return downloadZipAndChecksum(terminal, url, tmpDir, false);
+            return downloadAndValidate(terminal, url, tmpDir, true);
         }
 
         // now try as maven coordinates, a valid URL would only have a colon and slash
         String[] coordinates = pluginId.split(":");
         if (coordinates.length == 3 && pluginId.contains("/") == false && pluginId.startsWith("file:") == false) {
             String mavenUrl = getMavenUrl(terminal, coordinates, Platforms.PLATFORM_NAME);
             terminal.println("-> Downloading " + pluginId + " from maven central");
-            return downloadZipAndChecksum(terminal, mavenUrl, tmpDir, true);
+            return downloadAndValidate(terminal, mavenUrl, tmpDir, false);
         }
 
         // fall back to plain old URL
@@ -406,16 +419,44 @@ public void onProgress(int percent) {
         }
     }
 
-    /** Downloads a zip from the url, as well as a SHA512 (or SHA1) checksum, and checks the checksum. */
-    // pkg private for tests
-    @SuppressForbidden(reason = "We use openStream to download plugins")
-    private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir, boolean allowSha1) throws Exception {
+    @SuppressForbidden(reason = "URL#openStream")
+    private InputStream urlOpenStream(final URL url) throws IOException {
+        return url.openStream();
+    }
+
+    /**
+     * Downlaods a ZIP from the URL. This method also validates the downloaded plugin ZIP via the following means:
+     * <ul>
+     * <li>
+     * For an official plugin we download the SHA-512 checksum and validate the integrity of the downloaded ZIP. We also download the
+     * armored signature and validate the authenticity of the downloaded ZIP.
+     * </li>
+     * <li>
+     * For a non-official plugin we download the SHA-512 checksum and fallback to the SHA-1 checksum and validate the integrity of the
+     * downloaded ZIP.
+     * </li>
+     * </ul>
+     *
+     * @param terminal       a terminal to log messages to
+     * @param urlString      the URL of the plugin ZIP
+     * @param tmpDir         a temporary directory to write downloaded files to
+     * @param officialPlugin true if the plugin is an official plugin
+     * @return the path to the downloaded plugin ZIP
+     * @throws IOException   if an I/O exception occurs download or reading files and resources
+     * @throws PGPException  if an exception occurs verifying the downloaded ZIP signature
+     * @throws UserException if checksum validation fails
+     */
+    private Path downloadAndValidate(
+            final Terminal terminal,
+            final String urlString,
+            final Path tmpDir,
+            final boolean officialPlugin) throws IOException, PGPException, UserException {
         Path zip = downloadZip(terminal, urlString, tmpDir);
         pathsToDeleteOnShutdown.add(zip);
         String checksumUrlString = urlString + ".sha512";
         URL checksumUrl = openUrl(checksumUrlString);
         String digestAlgo = "SHA-512";
-        if (checksumUrl == null && allowSha1) {
+        if (checksumUrl == null && officialPlugin == false) {
             // fallback to sha1, until 7.0, but with warning
             terminal.println("Warning: sha512 not found, falling back to sha1. This behavior is deprecated and will be removed in a " +
                              "future release. Please update the plugin to use a sha512 checksum.");
@@ -427,7 +468,7 @@ private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tm
             throw new UserException(ExitCodes.IO_ERROR, "Plugin checksum missing: " + checksumUrlString);
         }
         final String expectedChecksum;
-        try (InputStream in = checksumUrl.openStream()) {
+        try (InputStream in = urlOpenStream(checksumUrl)) {
             /*
              * The supported format of the SHA-1 files is a single-line file containing the SHA-1. The supported format of the SHA-512 files
              * is a single-line file containing the SHA-512 and the filename, separated by two spaces. For SHA-1, we verify that the hash
@@ -465,23 +506,84 @@ private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tm
             }
         }
 
-        byte[] zipbytes = Files.readAllBytes(zip);
-        String gotChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipbytes));
-        if (expectedChecksum.equals(gotChecksum) == false) {
-            throw new UserException(ExitCodes.IO_ERROR,
-                digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
+        try {
+            final byte[] zipBytes = Files.readAllBytes(zip);
+            final String actualChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipBytes));
+            if (expectedChecksum.equals(actualChecksum) == false) {
+                throw new UserException(
+                        ExitCodes.IO_ERROR,
+                        digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + actualChecksum);
+            }
+        } catch (final NoSuchAlgorithmException e) {
+            // this should never happen as we are using SHA-1 and SHA-512 here
+            throw new AssertionError(e);
+        }
+
+        if (officialPlugin) {
+            verifySignature(zip, urlString);
         }
 
         return zip;
     }
 
+    static {
+        Security.addProvider(new BouncyCastleProvider());
+    }
+
+    void verifySignature(final Path zip, final String urlString) throws IOException, PGPException {
+        final String ascUrlString = urlString + ".asc";
+        final URL ascUrl = openUrl(ascUrlString);
+        try (// fin is a file stream over the downloaded plugin zip whose signature to verify
+             InputStream fin = pluginZipInputStream(zip);
+             // sin is a URL stream to the signature corresponding to the downloaded plugin zip
+             InputStream sin = urlOpenStream(ascUrl);
+             // pin is a decoded base64 stream over the embedded public key in RFC2045 format
+             InputStream pin = Base64.getMimeDecoder().wrap(getPublicKey())) {
+            final JcaPGPObjectFactory factory = new JcaPGPObjectFactory(PGPUtil.getDecoderStream(sin));
+            final PGPSignature signature = ((PGPSignatureList) factory.nextObject()).get(0);
+
+            // validate the signature has key ID matching our public key ID
+            final String keyId = Long.toHexString(signature.getKeyID()).toUpperCase(Locale.ROOT);
+            if (getPublicKeyId().equals(keyId) == false) {
+                throw new IllegalStateException("key id [" + keyId + "] does not match expected key id [" + getPublicKeyId() + "]");
+            }
+
+            // compute the signature of the downloaded plugin zip
+            final PGPPublicKeyRingCollection collection = new PGPPublicKeyRingCollection(pin, new JcaKeyFingerprintCalculator());
+            final PGPPublicKey key = collection.getPublicKey(signature.getKeyID());
+            signature.init(new JcaPGPContentVerifierBuilderProvider().setProvider("BC"), key);
+            final byte[] buffer = new byte[1024];
+            int read;
+            while ((read = fin.read(buffer)) != -1) {
+                signature.update(buffer, 0, read);
+            }
+
+            // finally we verify the signature of the downloaded plugin zip matches the expected signature
+            if (signature.verify() == false) {
+                throw new IllegalStateException("signature verification for [" + urlString + "] failed");
+            }
+        }
+    }
+
+    InputStream pluginZipInputStream(final Path zip) throws IOException {
+        return Files.newInputStream(zip);
+    }
+
+    String getPublicKeyId() {
+        return "D27D666CD88E42B4";
+    }
+
+    InputStream getPublicKey() {
+        return InstallPluginCommand.class.getResourceAsStream("/public_key");
+    }
+
     /**
      * Creates a URL and opens a connection.
      *
      * If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned.
      */
     // pkg private for tests
-    URL openUrl(String urlString) throws Exception {
+    URL openUrl(String urlString) throws IOException {
         URL checksumUrl = new URL(urlString);
         HttpURLConnection connection = (HttpURLConnection)checksumUrl.openConnection();
         if (connection.getResponseCode() == 404) {
@@ -605,11 +707,21 @@ private PluginInfo loadPluginInfo(Terminal terminal, Path pluginRoot, Environmen
         return info;
     }
 
+    private static final String LIB_TOOLS_PLUGIN_CLI_CLASSPATH_JAR;
+
+    static {
+        LIB_TOOLS_PLUGIN_CLI_CLASSPATH_JAR =
+                String.format(Locale.ROOT, ".+%1$slib%1$stools%1$splugin-cli%1$s[^%1$s]+\\.jar", "/|\\\\");
+    }
+
     /** check a candidate plugin for jar hell before installing it */
     void jarHellCheck(PluginInfo candidateInfo, Path candidateDir, Path pluginsDir, Path modulesDir) throws Exception {
         // create list of current jars in classpath
-        final Set<URL> jars = new HashSet<>(JarHell.parseClassPath());
-
+        final Set<URL> classpath =
+                JarHell.parseClassPath()
+                        .stream()
+                        .filter(url -> urlToString(url).matches(LIB_TOOLS_PLUGIN_CLI_CLASSPATH_JAR) == false)
+                        .collect(Collectors.toSet());
 
         // read existing bundles. this does some checks on the installation too.
         Set<PluginsService.Bundle> bundles = new HashSet<>(PluginsService.getPluginBundles(pluginsDir));
@@ -621,13 +733,21 @@ void jarHellCheck(PluginInfo candidateInfo, Path candidateDir, Path pluginsDir,
         // TODO: optimize to skip any bundles not connected to the candidate plugin?
         Map<String, Set<URL>> transitiveUrls = new HashMap<>();
         for (PluginsService.Bundle bundle : sortedBundles) {
-            PluginsService.checkBundleJarHell(bundle, transitiveUrls);
+            PluginsService.checkBundleJarHell(classpath, bundle, transitiveUrls);
         }
 
         // TODO: no jars should be an error
         // TODO: verify the classname exists in one of the jars!
     }
 
+    private String urlToString(final URL url) {
+        try {
+            return url.toURI().getPath();
+        } catch (final URISyntaxException e) {
+            throw new AssertionError(e);
+        }
+    }
+
     private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environment env) throws Exception {
         List<Path> deleteOnFailure = new ArrayList<>();
         deleteOnFailure.add(tmpRoot);

distribution/tools/plugin-cli/src/main/resources/public_key

@@ -0,0 +1,24 @@
+mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBDA+bGFOwy
+hbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9CUliQe324qvObU2Q
+RtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZj3SF1SPO+TB5QrHkrQHBsmX+
+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj
+1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEB
+AAG0RUVsYXN0aWNzZWFyY2ggKEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3Bz
+QGVsYXN0aWNzZWFyY2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYC
+AwECHgECF4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75
+nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/7C2GuGCO
+lbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKmTxcDTFrV7SmVPxCB
+cQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe8d7sw+XvxB2aN4gnTlRzjL1n
+TRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3
+vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUlzcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNM
+KGTABFG1yRx9r+wa/fvqP6OTRzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hp
+lnpU+PBQZJ5XJ2I+1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA
+07xx7Bj+Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt
+KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0KwwEwSk/UDu
+ToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0c3MIAIE9hAR20mqJ
+WLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12JTavnJ5MLaETlggXY+zDef9sy
+TPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZ
+EyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWg
+R7U2r+a210W6vnUxU4oN0PmMcursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNt
+fllxIu9XYmiBERQ/qPDlGRlOgVTd9xUfHFkzB52c70E=
+=92oX