Skip to content

Commit

Permalink
Plugins: Add backcompat for sha1 checksums (#26748)
Browse files Browse the repository at this point in the history 
With 6.0 rc1 we now publish sha512 checksums for official plugins.
However, in order to ease the pain for plugin authors, this commit adds
backcompat to still allow sha1 checksums.  Also added tests for
checksums.

Closes #26746
  • Loading branch information
@rjernst @jasontedor
rjernst authored and jasontedor committed Sep 22, 2017
1 parent eab7b25 commit 8e83bfa
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 12 deletions.
43 changes: 35 additions & 8 deletions ...bution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -222,15 +223,15 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Ex
if (OFFICIAL_PLUGINS.contains(pluginId)) {
final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME);
terminal.println("-> Downloading " + pluginId + " from elastic");
return downloadZipAndChecksum(terminal, url, tmpDir);
return downloadZipAndChecksum(terminal, url, tmpDir, false);
}

// now try as maven coordinates, a valid URL would only have a colon and slash
String[] coordinates = pluginId.split(":");
if (coordinates.length == 3 && pluginId.contains("/") == false) {
String mavenUrl = getMavenUrl(terminal, coordinates, Platforms.PLATFORM_NAME);
terminal.println("-> Downloading " + pluginId + " from maven central");
return downloadZipAndChecksum(terminal, mavenUrl, tmpDir);
return downloadZipAndChecksum(terminal, mavenUrl, tmpDir, true);
}

// fall back to plain old URL
Expand Down Expand Up @@ -316,7 +317,8 @@ private List<String> checkMisspelledPlugin(String pluginId) {
}

/** Downloads a zip from the url, into a temp file under the given temp dir. */
private Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
@SuppressForbidden(reason = "We use getInputStream to download plugins")
Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
terminal.println(VERBOSE, "Retrieving zip from " + urlString);
URL url = new URL(urlString);
Path zip = Files.createTempFile(tmpDir, null, ".zip");
Expand Down Expand Up @@ -364,13 +366,23 @@ public void onProgress(int percent) {
}
}

/** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */
/** Downloads a zip from the url, as well as a SHA512 (or SHA1) checksum, and checks the checksum. */
// pkg private for tests
@SuppressForbidden(reason = "We use openStream to download plugins")
Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception {
private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir, boolean allowSha1) throws Exception {
Path zip = downloadZip(terminal, urlString, tmpDir);
pathsToDeleteOnShutdown.add(zip);
URL checksumUrl = new URL(urlString + ".sha1");
String checksumUrlString = urlString + ".sha512";
URL checksumUrl = openUrl(checksumUrlString);
String digestAlgo = "SHA-512";
if (checksumUrl == null && allowSha1) {
checksumUrlString = urlString + ".sha1";
checksumUrl = openUrl(checksumUrlString);
digestAlgo = "SHA-1";
}
if (checksumUrl == null) {
throw new UserException(ExitCodes.IO_ERROR, "Plugin checksum missing: " + checksumUrlString);
}
final String expectedChecksum;
try (InputStream in = checksumUrl.openStream()) {
BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
Expand All @@ -381,15 +393,30 @@ Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) th
}

byte[] zipbytes = Files.readAllBytes(zip);
String gotChecksum = MessageDigests.toHexString(MessageDigests.sha1().digest(zipbytes));
String gotChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipbytes));
if (expectedChecksum.equals(gotChecksum) == false) {
throw new UserException(ExitCodes.IO_ERROR,
"SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
}

return zip;
}

/**
* Creates a URL and opens a connection.
*
* If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned.
*/
// pkg private for tests
URL openUrl(String urlString) throws Exception {
URL checksumUrl = new URL(urlString);
HttpURLConnection connection = (HttpURLConnection)checksumUrl.openConnection();
if (connection.getResponseCode() == 404) {
return null;
}
return checksumUrl;
}

private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException {
// unzip plugin to a staging temp dir

Expand Down
84 changes: 80 additions & 4 deletions ...n/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@
import com.google.common.jimfs.Jimfs;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.Version;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cli.UserException;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.io.PathUtilsForTesting;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -60,6 +62,7 @@
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.UserPrincipal;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -753,19 +756,33 @@ private void installPlugin(MockTerminal terminal, boolean isBatch) throws Except
skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2());
}

public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception {
public MockTerminal assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash,
String shaExtension, Function<byte[], String> shaCalculator) throws Exception {
Tuple<Path, Environment> env = createEnv(fs, temp);
Path pluginDir = createPluginDir(temp);
Path pluginZip = createPlugin(name, pluginDir, false);
InstallPluginCommand command = new InstallPluginCommand() {
@Override
Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception {
Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
assertEquals(url, urlString);
Path downloadedPath = tmpDir.resolve("downloaded.zip");
Files.copy(pluginZip, downloadedPath);
return downloadedPath;
}
@Override
URL openUrl(String urlString) throws Exception {
String expectedUrl = url + shaExtension;
if (expectedUrl.equals(urlString)) {
// calc sha an return file URL to it
Path shaFile = temp.apply("shas").resolve("downloaded.zip" + shaExtension);
byte[] zipbytes = Files.readAllBytes(pluginZip);
String checksum = shaCalculator.apply(zipbytes);
Files.write(shaFile, checksum.getBytes(StandardCharsets.UTF_8));
return shaFile.toUri().toURL();
}
return null;
}
@Override
boolean urlExists(Terminal terminal, String urlString) throws IOException {
return urlString.equals(url);
}
Expand All @@ -778,8 +795,15 @@ void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
// no jarhell check
}
};
installPlugin(pluginId, env.v1(), command);
MockTerminal terminal = installPlugin(pluginId, env.v1(), command);
assertPlugin(name, pluginDir, env.v2());
return terminal;
}

public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-512");
assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512",
bytes -> MessageDigests.toHexString(digest.digest(bytes)));
}

public void testOfficalPlugin() throws Exception {
Expand Down Expand Up @@ -815,5 +839,57 @@ public void testMavenPlatformPlugin() throws Exception {
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
}

// TODO: test checksum (need maven/official below)
public void testMavenSha1Backcompat() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
MessageDigest digest = MessageDigest.getInstance("SHA-1");
MockTerminal terminal = assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null,
".sha1", bytes -> MessageDigests.toHexString(digest.digest(bytes)));
}

public void testOfficialShaMissing() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
MessageDigest digest = MessageDigest.getInstance("SHA-1");
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1",
bytes -> MessageDigests.toHexString(digest.digest(bytes))));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage());
}

public void testMavenShaMissing() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage());
}

public void testInvalidShaFile() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
MessageDigest digest = MessageDigest.getInstance("SHA-512");
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512",
bytes -> MessageDigests.toHexString(digest.digest(bytes)) + "\nfoobar"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file"));
}

public void testSha512Mismatch() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512",
bytes -> "foobar"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().contains("SHA-512 mismatch, expected foobar"));
}

public void testSha1Mismatch() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null,
".sha1", bytes -> "foobar"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar"));
}

}

0 comments on commit 8e83bfa

Please sign in to comment.