Skip to content

Commit

Permalink
Merge pull request #16843 from xuzha/s3-encryption
Browse files Browse the repository at this point in the history 
S3 client side encryption
  • Loading branch information
@xuzha
xuzha committed Mar 24, 2016
2 parents 08903f1 + 38923b8 commit 37a183d
Show file tree
Hide file tree
Showing 10 changed files with 252 additions and 35 deletions.
10 changes: 10 additions & 0 deletions docs/plugins/repository-s3.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,16 @@ The following settings are supported:
currently supported by the plugin. For more information about the
different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]

`client_symmetric_key`::
Sets the keys to use to encrypt your snapshots. You can specify either a symmetric key or a public/private key pair.
No encryption by default. This sets a Base64-encoded AES symmetric-key (128, 192 or 256 bits)

`client_public_key`::
Sets the a base64-encoded RSA public key

`client_private_key`::
Sets the a base64-encoded RSA private key

The S3 repositories use the same credentials as the rest of the AWS services
provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.

Expand Down
3 changes: 2 additions & 1 deletion plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
Expand Down Expand Up @@ -153,5 +154,5 @@ interface CLOUD_S3 {
Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
}

AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials);
}
33 changes: 27 additions & 6 deletions plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
Expand All @@ -49,15 +54,17 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
/**
* (acceskey, endpoint) -&gt; client
*/
private Map<Tuple<String, String>, AmazonS3Client> clients = new HashMap<>();
private Map<Tuple<String, Tuple<String, EncryptionMaterials>>, AmazonS3Client> clients = new HashMap<>();

@Inject
public InternalAwsS3Service(Settings settings) {
super(settings);
}

@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
EncryptionMaterials clientSideEncryptionMaterials) {

if (Strings.isNullOrEmpty(endpoint)) {
// We need to set the endpoint based on the region
if (region != null) {
Expand All @@ -69,11 +76,14 @@ public synchronized AmazonS3 client(String endpoint, Protocol protocol, String r
}
}

return getClient(endpoint, protocol, account, key, maxRetries);
return getClient(endpoint, protocol, account, key, maxRetries, clientSideEncryptionMaterials);
}

private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) {
Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
EncryptionMaterials clientSideEncryptionMaterials) {

Tuple<String, EncryptionMaterials> tempTuple = new Tuple<>(account, clientSideEncryptionMaterials);
Tuple<String, Tuple<String, EncryptionMaterials>> clientDescriptor = new Tuple<>(endpoint, tempTuple);
AmazonS3Client client = clients.get(clientDescriptor);
if (client != null) {
return client;
Expand Down Expand Up @@ -123,7 +133,18 @@ private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, Stri
new StaticCredentialsProvider(new BasicAWSCredentials(account, key))
);
}
client = new AmazonS3Client(credentials, clientConfiguration);

if (clientSideEncryptionMaterials != null) {
EncryptionMaterialsProvider encryptionMaterialsProvider = new StaticEncryptionMaterialsProvider(clientSideEncryptionMaterials);
CryptoConfiguration cryptoConfiguration = new CryptoConfiguration();
client = new AmazonS3EncryptionClient(
credentials,
encryptionMaterialsProvider,
clientConfiguration,
cryptoConfiguration);
} else {
client = new AmazonS3Client(credentials, clientConfiguration);
}

if (endpoint != null) {
client.setEndpoint(endpoint);
Expand Down
25 changes: 2 additions & 23 deletions ...ository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,31 +131,10 @@ protected void doUpload(S3BlobStore blobStore, String bucketName, String blobNam
}
md.setContentLength(length);

InputStream inputStream = is;

// We try to compute a MD5 while reading it
MessageDigest messageDigest;
try {
messageDigest = MessageDigest.getInstance("MD5");
inputStream = new DigestInputStream(is, messageDigest);
} catch (NoSuchAlgorithmException impossible) {
// Every implementation of the Java platform is required to support MD5 (see MessageDigest)
throw new RuntimeException(impossible);
}

PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, inputStream, md)
PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, is, md)
.withStorageClass(blobStore.getStorageClass())
.withCannedAcl(blobStore.getCannedACL());
PutObjectResult putObjectResult = blobStore.client().putObject(putRequest);

String localMd5 = Base64.encodeAsString(messageDigest.digest());
String remoteMd5 = putObjectResult.getContentMd5();
if (!localMd5.equals(remoteMd5)) {
logger.debug("MD5 local [{}], remote [{}] are not equal...", localMd5, remoteMd5);
throw new AmazonS3Exception("MD5 local [" + localMd5 +
"], remote [" + remoteMd5 +
"] are not equal...");
}
blobStore.client().putObject(putRequest);
}

private void initializeMultipart() {
Expand Down
7 changes: 7 additions & 0 deletions plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CreateBucketRequest;
Expand Down Expand Up @@ -70,6 +71,12 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable
this.region = region;
this.serverSideEncryption = serverSideEncryption;
this.bufferSize = bufferSize;

if (client instanceof AmazonS3EncryptionClient && this.bufferSize.getBytes() % 16 > 0) {
throw new BlobStoreException("Detected client-side encryption " +
"and a buffer_size for the S3 storage not a multiple of the cipher block size (16)");
}

this.cannedACL = initCannedACL(cannedACL);
this.numberOfRetries = maxRetries;
this.storageClass = initStorageClass(storageClass);
Expand Down
6 changes: 6 additions & 0 deletions ...epository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,9 @@ public void onModule(SettingsModule settingsModule) {
settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PRIVATE_KEY);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PUBLIC_KEY);
settingsModule.registerSetting(S3Repository.Repositories.CLIENT_SYMMETRIC_KEY);

// Register S3 single repository settings
settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
Expand All @@ -144,6 +147,9 @@ public void onModule(SettingsModule settingsModule) {
settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PRIVATE_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_PUBLIC_KEY);
settingsModule.registerSetting(S3Repository.Repository.CLIENT_SYMMETRIC_KEY);
}

/**
Expand Down
109 changes: 107 additions & 2 deletions plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
package org.elasticsearch.repositories.s3;

import com.amazonaws.Protocol;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import com.amazonaws.util.Base64;
import org.elasticsearch.cloud.aws.AwsS3Service;
import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
Expand All @@ -37,7 +39,15 @@
import org.elasticsearch.repositories.RepositorySettings;
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;

import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.security.KeyFactory;
import java.security.KeyPair;
import java.security.NoSuchAlgorithmException;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.util.Locale;
import java.util.function.Function;

Expand Down Expand Up @@ -140,6 +150,21 @@ public interface Repositories {
* repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory.
*/
Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope);
/**
* repositories.s3.client_symmetric_key: Specifies the Base64-encoded AES symmetric-key (128, 192 or 256 bits)
*/
Setting<String> CLIENT_SYMMETRIC_KEY = Setting.simpleString("repositories.s3.client_symmetric_key", Property.NodeScope);

/**
* repositories.s3.client_public_key: Specifies the Base64-encoded RSA public key
*/
Setting<String> CLIENT_PUBLIC_KEY = Setting.simpleString("repositories.s3.client_public_key", Property.NodeScope);

/**
* repositories.s3.client_private_key: Specifies the Base64-encoded RSA private key
*/
Setting<String> CLIENT_PRIVATE_KEY = Setting.simpleString("repositories.s3.client_private_key", Property.NodeScope);

}

/**
Expand Down Expand Up @@ -223,6 +248,24 @@ public interface Repository {
* @see Repositories#BASE_PATH_SETTING
*/
Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);

/**
* base_path
* @see Repositories#CLIENT_SYMMETRIC_KEY
*/
Setting<String> CLIENT_SYMMETRIC_KEY = Setting.simpleString("client_symmetric_key", Property.NodeScope);

/**
* base_path
* @see Repositories#CLIENT_PUBLIC_KEY
*/
Setting<String> CLIENT_PUBLIC_KEY = Setting.simpleString("client_public_key", Property.NodeScope);

/**
* base_path
* @see Repositories#CLIENT_PRIVATE_KEY
*/
Setting<String> CLIENT_PRIVATE_KEY = Setting.simpleString("client_private_key", Property.NodeScope);
}

private final S3BlobStore blobStore;
Expand Down Expand Up @@ -281,8 +324,24 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings,
String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);

blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries),
bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
// parse and validate the client side encryption setting
String symmetricKeyBase64 = getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY);
String publicKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY);
String privateKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY);

EncryptionMaterials clientSideEncryptionMaterials = initClientSideEncryption(symmetricKeyBase64, publicKeyBase64, privateKeyBase64, name);

blobStore = new S3BlobStore(
settings,
s3Service.client(endpoint, protocol, region, key, secret, maxRetries, clientSideEncryptionMaterials),
bucket,
region,
serverSideEncryption,
bufferSize,
maxRetries,
cannedACL,
storageClass
);

String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
if (Strings.hasLength(basePath)) {
Expand All @@ -294,6 +353,52 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings,
} else {
this.basePath = BlobPath.cleanPath();
}

}


/**
* Init and verify initClientSideEncryption settings
*/
private EncryptionMaterials initClientSideEncryption(String symmetricKey, String publicKey, String privateKey, RepositoryName name) {

EncryptionMaterials clientSideEncryptionMaterials = null;

if (Strings.isNullOrEmpty(symmetricKey) == false && (Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false)) {
throw new RepositoryException(name.name(), "Client-side encryption: You can't specify a symmetric key AND a public/private key pair");
}

if (Strings.isNullOrEmpty(symmetricKey) == false || Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false) {
try {
// Check crypto
if (Cipher.getMaxAllowedKeyLength("AES") < 256) {
throw new RepositoryException(name.name(), "Client-side encryption: Please install the Java Cryptography Extension");
}

// Transform the keys in a EncryptionMaterials
if (Strings.isNullOrEmpty(symmetricKey) == false) {
clientSideEncryptionMaterials = new EncryptionMaterials(new SecretKeySpec(Base64.decode(symmetricKey), "AES"));
} else {
if (Strings.isNullOrEmpty(publicKey) || Strings.isNullOrEmpty(privateKey)) {
String missingKey = Strings.isNullOrEmpty(publicKey) ? "public key" : "private key";
throw new RepositoryException(name.name(), "Client-side encryption: " + missingKey + " is missing");
}

clientSideEncryptionMaterials = new EncryptionMaterials(new KeyPair(
KeyFactory.getInstance("RSA").generatePublic(new X509EncodedKeySpec(Base64.decode(publicKey))),
KeyFactory.getInstance("RSA").generatePrivate(new PKCS8EncodedKeySpec(Base64.decode(privateKey)))));
}

} catch (IllegalArgumentException e) {
throw new RepositoryException(name.name(), "Client-side encryption: Error decoding your keys: " + e.getMessage());
} catch (NoSuchAlgorithmException e) {
throw new RepositoryException(name.name(), e.getMessage());
} catch (InvalidKeySpecException e) {
throw new RepositoryException(name.name(), e.getMessage());
}
}

return clientSideEncryptionMaterials;
}

/**
Expand Down
8 changes: 8 additions & 0 deletions ...ns/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
.put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class")
.put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl")
.put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath")

.build();

/**
Expand Down Expand Up @@ -125,6 +126,9 @@ public void testRepositorySettingsGlobalOnly() {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
}

/**
Expand Down Expand Up @@ -153,6 +157,9 @@ public void testRepositorySettingsGlobalOverloadedByS3() {
assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
}

/**
Expand Down Expand Up @@ -329,6 +336,7 @@ private Settings buildSettings(Settings... global) {

private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage)
throws IOException {

Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder()
.put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer)
Expand Down
5 changes: 3 additions & 2 deletions plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.EncryptionMaterials;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -51,8 +52,8 @@ public TestAwsS3Service(Settings settings) {


@Override
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) {
return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, clientSideEncryptionMaterials));
}

private AmazonS3 cachedWrapper(AmazonS3 client) {
Expand Down
Loading

0 comments on commit 37a183d

Please sign in to comment.