From 5b711c283dbc233f80de5e0adb26723c22d678c7 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 22 Sep 2017 02:26:32 -0700 Subject: [PATCH] Plugins: Add backcompat for sha1 checksums (#26748) With 6.0 rc1 we now publish sha512 checksums for official plugins. However, in order to ease the pain for plugin authors, this commit adds backcompat to still allow sha1 checksums. Also added tests for checksums. Closes #26746 --- .../plugins/InstallPluginCommand.java | 46 ++++++++-- .../plugins/InstallPluginCommandTests.java | 85 +++++++++++++++++-- 2 files changed, 118 insertions(+), 13 deletions(-) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 1ccb6f740ddd4..ec017cfffb5b6 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -58,6 +58,7 @@ import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -218,7 +219,7 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Ex if (OFFICIAL_PLUGINS.contains(pluginId)) { final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME); terminal.println("-> Downloading " + pluginId + " from elastic"); - return downloadZipAndChecksum(terminal, url, tmpDir); + return downloadZipAndChecksum(terminal, url, tmpDir, false); } // now try as maven coordinates, a valid URL would only have a colon and slash @@ -226,7 +227,7 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Ex if (coordinates.length == 3 && pluginId.contains("/") == false) { String mavenUrl = getMavenUrl(terminal, coordinates, Platforms.PLATFORM_NAME); terminal.println("-> Downloading " + pluginId + " from maven central"); - return downloadZipAndChecksum(terminal, mavenUrl, tmpDir); + return downloadZipAndChecksum(terminal, mavenUrl, tmpDir, true); } // fall back to plain old URL @@ -312,8 +313,9 @@ private List checkMisspelledPlugin(String pluginId) { } /** Downloads a zip from the url, into a temp file under the given temp dir. */ + // pkg private for tests @SuppressForbidden(reason = "We use getInputStream to download plugins") - private Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException { + Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException { terminal.println(VERBOSE, "Retrieving zip from " + urlString); URL url = new URL(urlString); Path zip = Files.createTempFile(tmpDir, null, ".zip"); @@ -361,13 +363,26 @@ public void onProgress(int percent) { } } - /** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */ + /** Downloads a zip from the url, as well as a SHA512 (or SHA1) checksum, and checks the checksum. */ // pkg private for tests @SuppressForbidden(reason = "We use openStream to download plugins") - Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception { + private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir, boolean allowSha1) throws Exception { Path zip = downloadZip(terminal, urlString, tmpDir); pathsToDeleteOnShutdown.add(zip); - URL checksumUrl = new URL(urlString + ".sha1"); + String checksumUrlString = urlString + ".sha512"; + URL checksumUrl = openUrl(checksumUrlString); + String digestAlgo = "SHA-512"; + if (checksumUrl == null && allowSha1) { + // fallback to sha1, until 7.0, but with warning + terminal.println("Warning: sha512 not found, falling back to sha1. This behavior is deprecated and will be removed in a " + + "future release. Please update the plugin to use a sha512 checksum."); + checksumUrlString = urlString + ".sha1"; + checksumUrl = openUrl(checksumUrlString); + digestAlgo = "SHA-1"; + } + if (checksumUrl == null) { + throw new UserException(ExitCodes.IO_ERROR, "Plugin checksum missing: " + checksumUrlString); + } final String expectedChecksum; try (InputStream in = checksumUrl.openStream()) { BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); @@ -378,15 +393,30 @@ Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) th } byte[] zipbytes = Files.readAllBytes(zip); - String gotChecksum = MessageDigests.toHexString(MessageDigests.sha1().digest(zipbytes)); + String gotChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipbytes)); if (expectedChecksum.equals(gotChecksum) == false) { throw new UserException(ExitCodes.IO_ERROR, - "SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum); + digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + gotChecksum); } return zip; } + /** + * Creates a URL and opens a connection. + * + * If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned. + */ + // pkg private for tests + URL openUrl(String urlString) throws Exception { + URL checksumUrl = new URL(urlString); + HttpURLConnection connection = (HttpURLConnection)checksumUrl.openConnection(); + if (connection.getResponseCode() == 404) { + return null; + } + return checksumUrl; + } + private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException { // unzip plugin to a staging temp dir diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index d8166a348f64b..103baccd2d431 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -24,11 +24,13 @@ import com.google.common.jimfs.Jimfs; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; +import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; @@ -62,7 +64,7 @@ import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.UserPrincipal; -import java.security.KeyStore; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -751,19 +753,33 @@ private void installPlugin(MockTerminal terminal, boolean isBatch) throws Except skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2()); } - public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception { + public MockTerminal assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash, + String shaExtension, Function shaCalculator) throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); Path pluginZip = createPlugin(name, pluginDir, false); InstallPluginCommand command = new InstallPluginCommand() { @Override - Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception { + Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException { assertEquals(url, urlString); Path downloadedPath = tmpDir.resolve("downloaded.zip"); Files.copy(pluginZip, downloadedPath); return downloadedPath; } @Override + URL openUrl(String urlString) throws Exception { + String expectedUrl = url + shaExtension; + if (expectedUrl.equals(urlString)) { + // calc sha an return file URL to it + Path shaFile = temp.apply("shas").resolve("downloaded.zip" + shaExtension); + byte[] zipbytes = Files.readAllBytes(pluginZip); + String checksum = shaCalculator.apply(zipbytes); + Files.write(shaFile, checksum.getBytes(StandardCharsets.UTF_8)); + return shaFile.toUri().toURL(); + } + return null; + } + @Override boolean urlExists(Terminal terminal, String urlString) throws IOException { return urlString.equals(url); } @@ -776,8 +792,15 @@ void jarHellCheck(Path candidate, Path pluginsDir) throws Exception { // no jarhell check } }; - installPlugin(pluginId, env.v1(), command); + MockTerminal terminal = installPlugin(pluginId, env.v1(), command); assertPlugin(name, pluginDir, env.v2()); + return terminal; + } + + public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception { + MessageDigest digest = MessageDigest.getInstance("SHA-512"); + assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512", + bytes -> MessageDigests.toHexString(digest.digest(bytes))); } public void testOfficalPlugin() throws Exception { @@ -813,7 +836,59 @@ public void testMavenPlatformPlugin() throws Exception { assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null); } - // TODO: test checksum (need maven/official below) + public void testMavenSha1Backcompat() throws Exception { + String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + MockTerminal terminal = assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, + ".sha1", bytes -> MessageDigests.toHexString(digest.digest(bytes))); + assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1")); + } + + public void testOfficialShaMissing() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1", + bytes -> MessageDigests.toHexString(digest.digest(bytes)))); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage()); + } + + public void testMavenShaMissing() throws Exception { + String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null)); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage()); + } + + public void testInvalidShaFile() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-512"); + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", + bytes -> MessageDigests.toHexString(digest.digest(bytes)) + "\nfoobar")); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); + } + + public void testSha512Mismatch() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", + bytes -> "foobar")); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertTrue(e.getMessage(), e.getMessage().contains("SHA-512 mismatch, expected foobar")); + } + + public void testSha1Mismatch() throws Exception { + String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, + ".sha1", bytes -> "foobar")); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar")); + } public void testKeystoreNotRequired() throws Exception { Tuple env = createEnv(fs, temp);