Skip to content

Commit

Permalink
Plugins: Add backcompat for sha1 checksums (#26748)
Browse files Browse the repository at this point in the history 
With 6.0 rc1 we now publish sha512 checksums for official plugins.
However, in order to ease the pain for plugin authors, this commit adds
backcompat to still allow sha1 checksums.  Also added tests for
checksums.

Closes #26746
  • Loading branch information
@rjernst @s1monw
rjernst authored and s1monw committed Sep 22, 2017
1 parent ff737a8 commit 5b711c2
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 13 deletions.
46 changes: 38 additions & 8 deletions ...bution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -218,15 +219,15 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Ex
if (OFFICIAL_PLUGINS.contains(pluginId)) {
final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME);
terminal.println("-> Downloading " + pluginId + " from elastic");
return downloadZipAndChecksum(terminal, url, tmpDir);
return downloadZipAndChecksum(terminal, url, tmpDir, false);
}

// now try as maven coordinates, a valid URL would only have a colon and slash
String[] coordinates = pluginId.split(":");
if (coordinates.length == 3 && pluginId.contains("/") == false) {
String mavenUrl = getMavenUrl(terminal, coordinates, Platforms.PLATFORM_NAME);
terminal.println("-> Downloading " + pluginId + " from maven central");
return downloadZipAndChecksum(terminal, mavenUrl, tmpDir);
return downloadZipAndChecksum(terminal, mavenUrl, tmpDir, true);
}

// fall back to plain old URL
Expand Down Expand Up @@ -312,8 +313,9 @@ private List<String> checkMisspelledPlugin(String pluginId) {
}

/** Downloads a zip from the url, into a temp file under the given temp dir. */
// pkg private for tests
@SuppressForbidden(reason = "We use getInputStream to download plugins")
private Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
terminal.println(VERBOSE, "Retrieving zip from " + urlString);
URL url = new URL(urlString);
Path zip = Files.createTempFile(tmpDir, null, ".zip");
Expand Down Expand Up @@ -361,13 +363,26 @@ public void onProgress(int percent) {
}
}

/** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */
/** Downloads a zip from the url, as well as a SHA512 (or SHA1) checksum, and checks the checksum. */
// pkg private for tests
@SuppressForbidden(reason = "We use openStream to download plugins")
Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception {
private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir, boolean allowSha1) throws Exception {
Path zip = downloadZip(terminal, urlString, tmpDir);
pathsToDeleteOnShutdown.add(zip);
URL checksumUrl = new URL(urlString + ".sha1");
String checksumUrlString = urlString + ".sha512";
URL checksumUrl = openUrl(checksumUrlString);
String digestAlgo = "SHA-512";
if (checksumUrl == null && allowSha1) {
// fallback to sha1, until 7.0, but with warning
terminal.println("Warning: sha512 not found, falling back to sha1. This behavior is deprecated and will be removed in a " +
"future release. Please update the plugin to use a sha512 checksum.");
checksumUrlString = urlString + ".sha1";
checksumUrl = openUrl(checksumUrlString);
digestAlgo = "SHA-1";
}
if (checksumUrl == null) {
throw new UserException(ExitCodes.IO_ERROR, "Plugin checksum missing: " + checksumUrlString);
}
final String expectedChecksum;
try (InputStream in = checksumUrl.openStream()) {
BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
Expand All @@ -378,15 +393,30 @@ Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) th
}

byte[] zipbytes = Files.readAllBytes(zip);
String gotChecksum = MessageDigests.toHexString(MessageDigests.sha1().digest(zipbytes));
String gotChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipbytes));
if (expectedChecksum.equals(gotChecksum) == false) {
throw new UserException(ExitCodes.IO_ERROR,
"SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
}

return zip;
}

/**
* Creates a URL and opens a connection.
*
* If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned.
*/
// pkg private for tests
URL openUrl(String urlString) throws Exception {
URL checksumUrl = new URL(urlString);
HttpURLConnection connection = (HttpURLConnection)checksumUrl.openConnection();
if (connection.getResponseCode() == 404) {
return null;
}
return checksumUrl;
}

private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException {
// unzip plugin to a staging temp dir

Expand Down
85 changes: 80 additions & 5 deletions ...n/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@
import com.google.common.jimfs.Jimfs;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.Version;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cli.UserException;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.io.PathUtilsForTesting;
Expand Down Expand Up @@ -62,7 +64,7 @@
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.UserPrincipal;
import java.security.KeyStore;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
Expand Down Expand Up @@ -751,19 +753,33 @@ private void installPlugin(MockTerminal terminal, boolean isBatch) throws Except
skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2());
}

public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception {
public MockTerminal assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash,
String shaExtension, Function<byte[], String> shaCalculator) throws Exception {
Tuple<Path, Environment> env = createEnv(fs, temp);
Path pluginDir = createPluginDir(temp);
Path pluginZip = createPlugin(name, pluginDir, false);
InstallPluginCommand command = new InstallPluginCommand() {
@Override
Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception {
Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
assertEquals(url, urlString);
Path downloadedPath = tmpDir.resolve("downloaded.zip");
Files.copy(pluginZip, downloadedPath);
return downloadedPath;
}
@Override
URL openUrl(String urlString) throws Exception {
String expectedUrl = url + shaExtension;
if (expectedUrl.equals(urlString)) {
// calc sha an return file URL to it
Path shaFile = temp.apply("shas").resolve("downloaded.zip" + shaExtension);
byte[] zipbytes = Files.readAllBytes(pluginZip);
String checksum = shaCalculator.apply(zipbytes);
Files.write(shaFile, checksum.getBytes(StandardCharsets.UTF_8));
return shaFile.toUri().toURL();
}
return null;
}
@Override
boolean urlExists(Terminal terminal, String urlString) throws IOException {
return urlString.equals(url);
}
Expand All @@ -776,8 +792,15 @@ void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
// no jarhell check
}
};
installPlugin(pluginId, env.v1(), command);
MockTerminal terminal = installPlugin(pluginId, env.v1(), command);
assertPlugin(name, pluginDir, env.v2());
return terminal;
}

public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-512");
assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512",
bytes -> MessageDigests.toHexString(digest.digest(bytes)));
}

public void testOfficalPlugin() throws Exception {
Expand Down Expand Up @@ -813,7 +836,59 @@ public void testMavenPlatformPlugin() throws Exception {
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
}

// TODO: test checksum (need maven/official below)
public void testMavenSha1Backcompat() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
MessageDigest digest = MessageDigest.getInstance("SHA-1");
MockTerminal terminal = assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null,
".sha1", bytes -> MessageDigests.toHexString(digest.digest(bytes)));
assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1"));
}

public void testOfficialShaMissing() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
MessageDigest digest = MessageDigest.getInstance("SHA-1");
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1",
bytes -> MessageDigests.toHexString(digest.digest(bytes))));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage());
}

public void testMavenShaMissing() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage());
}

public void testInvalidShaFile() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
MessageDigest digest = MessageDigest.getInstance("SHA-512");
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512",
bytes -> MessageDigests.toHexString(digest.digest(bytes)) + "\nfoobar"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file"));
}

public void testSha512Mismatch() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512",
bytes -> "foobar"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().contains("SHA-512 mismatch, expected foobar"));
}

public void testSha1Mismatch() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null,
".sha1", bytes -> "foobar"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar"));
}

public void testKeystoreNotRequired() throws Exception {
Tuple<Path, Environment> env = createEnv(fs, temp);
Expand Down

0 comments on commit 5b711c2

Please sign in to comment.