From ea78fd6560624968fada645c6358eea84e4afb7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Tr=C3=A9segnie?= Date: Sat, 23 May 2015 09:17:35 +0100 Subject: [PATCH 1/3] Add client-side encryption The Java Cryptography Extension (JCE) has to be installed to use this feature. --- .../http/HttpServerTransport.java | 11 ++ .../cloud/aws/InternalAwsS3Service.java | 69 ++++++++- .../cloud/aws/blobstore/S3BlobStore.java | 10 ++ .../s3/AbstractS3SnapshotRestoreTest.java | 134 ++++++++++++++++++ 4 files changed, 222 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java index fa75f1dcb1eba..cacfeb144b140 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java @@ -19,6 +19,11 @@ package org.elasticsearch.http; +<<<<<<< HEAD:core/src/main/java/org/elasticsearch/http/HttpServerTransport.java +======= +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.EncryptionMaterials; +>>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -29,9 +34,15 @@ public interface HttpServerTransport extends LifecycleComponent>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 81b6463a746b9..e1c280f27ecae 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -31,6 +31,15 @@ import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; +<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +======= + +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.EncryptionMaterials; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; +>>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -47,9 +56,15 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { /** +<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java * (acceskey, endpoint) -> client */ private Map, AmazonS3Client> clients = new HashMap<>(); +======= + * (acceskey, (endpoint, clientSideEncryptionKey)) -> client + */ + private Map>, AmazonS3Client> clients = new HashMap>, AmazonS3Client>(); +>>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @Inject public InternalAwsS3Service(Settings settings) { @@ -57,6 +72,7 @@ public InternalAwsS3Service(Settings settings) { } @Override +<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) { if (Strings.isNullOrEmpty(endpoint)) { // We need to set the endpoint based on the region @@ -67,13 +83,50 @@ public synchronized AmazonS3 client(String endpoint, Protocol protocol, String r // No region has been set so we will use the default endpoint endpoint = getDefaultEndpoint(); } +======= + public synchronized AmazonS3 client() { + String endpoint = getDefaultEndpoint(); + String account = settings.get("cloud.aws.access_key", settings.get("cloud.account")); + String key = settings.get("cloud.aws.secret_key", settings.get("cloud.key")); + + return getClient(endpoint, null, account, key, null, null); + } + + @Override + public AmazonS3 client(String endpoint, String protocol, String region, String account, String key) { + return client(endpoint, protocol, region, account, key, null); + } + + @Override + public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) { + return client(endpoint, protocol, region, account, key, maxRetries, null); + } + + @Override + public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { + if (region != null && endpoint == null) { + endpoint = getEndpoint(region); + logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + } else if (endpoint == null) { + endpoint = getDefaultEndpoint(); + } + if (account == null || key == null) { + account = settings.get("cloud.aws.access_key", settings.get("cloud.account")); + key = settings.get("cloud.aws.secret_key", settings.get("cloud.key")); +>>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java } - return getClient(endpoint, protocol, account, key, maxRetries); + return getClient(endpoint, protocol, account, key, maxRetries, clientSideEncryptionMaterials); } +<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) { Tuple clientDescriptor = new Tuple<>(endpoint, account); +======= + + private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { + Tuple> clientDescriptor = new Tuple>(endpoint, new Tuple(account, clientSideEncryptionMaterials)); +>>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { return client; @@ -123,7 +176,19 @@ private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, Stri new StaticCredentialsProvider(new BasicAWSCredentials(account, key)) ); } - client = new AmazonS3Client(credentials, clientConfiguration); + + if(clientSideEncryptionMaterials != null) { + EncryptionMaterialsProvider encryptionMaterialsProvider = new StaticEncryptionMaterialsProvider(clientSideEncryptionMaterials); + CryptoConfiguration cryptoConfiguration = new CryptoConfiguration(); + client = new AmazonS3EncryptionClient( + credentials, + encryptionMaterialsProvider, + clientConfiguration, + cryptoConfiguration + ); + } else { + client = new AmazonS3Client(credentials, clientConfiguration); + } if (endpoint != null) { client.setEndpoint(endpoint); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index 650d71f62ad46..82eb9f63ee293 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -22,8 +22,12 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; +<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.CreateBucketRequest; +======= +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +>>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.ObjectListing; @@ -70,6 +74,12 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable this.region = region; this.serverSideEncryption = serverSideEncryption; this.bufferSize = bufferSize; + + if (client instanceof AmazonS3EncryptionClient && this.bufferSize.getBytes() % 16 > 0) { + throw new BlobStoreException("Detected client-side encryption " + + "and a buffer_size for the S3 storage not a multiple of the cipher block size (16)"); + } + this.cannedACL = initCannedACL(cannedACL); this.numberOfRetries = maxRetries; this.storageClass = initStorageClass(storageClass); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 793e442e119bf..e68a06c1ddfe1 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -23,7 +23,15 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +<<<<<<< HEAD:plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +======= + +import com.amazonaws.util.Base64; +import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.stream.JsonReader; +import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.stream.MalformedJsonException; +>>>>>>> 98d508f... Add client-side encryption:src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -33,6 +41,11 @@ import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; +<<<<<<< HEAD:plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +======= +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.repositories.RepositoryException; +>>>>>>> 98d508f... Add client-side encryption:src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryVerificationException; import org.elasticsearch.snapshots.SnapshotMissingException; @@ -42,7 +55,11 @@ import org.junit.After; import org.junit.Before; +import javax.crypto.KeyGenerator; +import java.io.InputStreamReader; +import java.security.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -118,6 +135,7 @@ public void testSimpleWorkflow() { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + assertMetadataFileIsNotEncrypted("test-snap"); logger.info("--> delete some data"); for (int i = 0; i < 50; i++) { @@ -249,6 +267,77 @@ public void testEncryption() { assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); } + @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") + public void testClientSideEncryption() throws NoSuchAlgorithmException { + + KeyGenerator keyGenerator1 = KeyGenerator.getInstance("AES"); + keyGenerator1.init(128); + String symmetricEncryptionKeyBase64 = Base64.encodeAsString(keyGenerator1.generateKey().getEncoded()); + + KeyPairGenerator keyGenerator2 = KeyPairGenerator.getInstance("RSA"); + keyGenerator2.initialize(512, new SecureRandom()); + KeyPair keyPair = keyGenerator2.generateKeyPair(); + String publicEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPublic().getEncoded()); + String privateEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPrivate().getEncoded()); + + Client client = client(); + try { + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64) + .put("client_side_encryption_key.public", publicEncryptionKeyBase64) + .put("client_side_encryption_key.private", privateEncryptionKeyBase64) + .put("chunk_size", randomIntBetween(1000, 10000)) + ).get(); + fail("Symmetric and public/private key pairs are exclusive options. An exception should be thrown."); + } catch(RepositoryException e) { + } + + List allSettings = Arrays.asList( + ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64) + .put("chunk_size", randomIntBetween(1000, 10000)), + ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("client_side_encryption_key.public", publicEncryptionKeyBase64) + .put("client_side_encryption_key.private", privateEncryptionKeyBase64) + .put("chunk_size", randomIntBetween(1000, 10000)) + ); + for(ImmutableSettings.Builder settings: allSettings) { + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(settings).get(); + + // Create the index and index some data + createIndex("test-idx-1"); + for (int i = 0; i < 100; i++) { + index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + // Take the snapshot + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + assertMetadataFileIsEncrypted("test-snap"); + + // Restore + cluster().wipeIndices("test-idx-1"); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet(); + ensureGreen(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); + assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); + + // Clean, the test will bbe run with different settings + cluster().wipeIndices("test-idx-1"); + wipeRepositories(); + cleanRepositoryFiles(basePath); + } + } + /** * This test verifies that the test configuration is set up in a manner that * does not make the test {@link #testRepositoryWithCustomCredentials()} pointless. @@ -435,6 +524,51 @@ private void assertRepositoryIsOperational(Client client, String repository) { assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); } + private void assertMetadataFileIsEncrypted(String snapshotName) { + + Settings settings = internalCluster().getInstance(Settings.class); + AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( + settings.get("repositories.s3.endpoint"), + settings.get("repositories.s3.protocol"), + settings.get("repositories.s3.region"), + settings.get("cloud.aws.access_key"), + settings.get("cloud.aws.secret_key")); + String bucket = settings.get("repositories.s3.bucket"); + String objectKey = basePath + "/metadata-" + snapshotName; + S3Object object = s3Client.getObject(bucket, objectKey); + + try { + JsonReader jsonReader = new JsonReader(new InputStreamReader(object.getObjectContent())); + jsonReader.beginObject(); + assertThat("The file hasn't been encrypted properly, its content is still readable!", jsonReader.nextName(), not(equalTo("meta-data"))); + } catch(Exception e) { + // The json is not valid, the file is encrypted + + // MalformedJsonException can't be catched directly so the following + // assertion is necessary to avoid silent failures. + assertThat(e, instanceOf(MalformedJsonException.class)); + } + } + + private void assertMetadataFileIsNotEncrypted(String snapshotName) { + + Settings settings = internalCluster().getInstance(Settings.class); + AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( + settings.get("repositories.s3.endpoint"), + settings.get("repositories.s3.protocol"), + settings.get("repositories.s3.region"), + settings.get("cloud.aws.access_key"), + settings.get("cloud.aws.secret_key")); + String bucket = settings.get("repositories.s3.bucket"); + String objectKey = basePath + "/metadata-" + snapshotName; + S3Object object = s3Client.getObject(bucket, objectKey); + + JsonReader jsonReader = new JsonReader(new InputStreamReader(object.getObjectContent())); + jsonReader.beginObject(); + assertThat("The file wasn't decrypted properly", jsonReader.nextName(), equalTo("meta-data")); + + // The beginning of the file looks like json. If it was encrypted, it wouldn't. + } /** * Deletes repositories, supports wildcard notation. From 7499e3aa4ac55cd79d5fe3276c293b9bfd2b3e13 Mon Sep 17 00:00:00 2001 From: Xu Zhang Date: Thu, 25 Feb 2016 17:13:31 -0800 Subject: [PATCH 2/3] Update and rebase the init implementation. Also removes the MD5 checks from our side, AWS S3 SDK java is doing the check. --- .../http/HttpServerTransport.java | 11 - docs/plugins/repository-s3.asciidoc | 10 + .../elasticsearch/cloud/aws/AwsS3Service.java | 2 + .../cloud/aws/InternalAwsS3Service.java | 62 +----- .../aws/blobstore/DefaultS3OutputStream.java | 27 +-- .../cloud/aws/blobstore/S3BlobStore.java | 7 +- .../repository/s3/S3RepositoryPlugin.java | 7 + .../repositories/s3/S3Repository.java | 112 +++++++++- .../s3/AbstractS3SnapshotRestoreTest.java | 204 +++++++----------- 9 files changed, 219 insertions(+), 223 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java index cacfeb144b140..fa75f1dcb1eba 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java @@ -19,11 +19,6 @@ package org.elasticsearch.http; -<<<<<<< HEAD:core/src/main/java/org/elasticsearch/http/HttpServerTransport.java -======= -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.EncryptionMaterials; ->>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -34,15 +29,9 @@ public interface HttpServerTransport extends LifecycleComponent>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java } diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 8780b3a710e74..af400dcfcfccb 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -236,6 +236,16 @@ The following settings are supported: currently supported by the plugin. For more information about the different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] +`client_symmetric_key`:: + Sets the keys to use to encrypt your snapshots. You can specify either a symmetric key or a public/private key pair. + No encryption by default. This sets a Base64-encoded AES symmetric-key (128, 192 or 256 bits) + +`client_public_key`:: + Sets the a base64-encoded RSA public key + +`client_private_key`:: + Sets the a base64-encoded RSA private key + The S3 repositories use the same credentials as the rest of the AWS services provided by this plugin (`discovery`). See <> for details. diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index 427c454fa280b..4b98176a14a77 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -21,6 +21,7 @@ import com.amazonaws.Protocol; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.EncryptionMaterials; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -154,4 +155,5 @@ interface CLOUD_S3 { } AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries); + AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index e1c280f27ecae..30a48dddc14f5 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -31,15 +31,11 @@ import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; -<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java -======= - import com.amazonaws.services.s3.AmazonS3EncryptionClient; import com.amazonaws.services.s3.model.CryptoConfiguration; import com.amazonaws.services.s3.model.EncryptionMaterials; import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; ->>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -56,15 +52,9 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { /** -<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java * (acceskey, endpoint) -> client */ - private Map, AmazonS3Client> clients = new HashMap<>(); -======= - * (acceskey, (endpoint, clientSideEncryptionKey)) -> client - */ - private Map>, AmazonS3Client> clients = new HashMap>, AmazonS3Client>(); ->>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java + private Map>, AmazonS3Client> clients = new HashMap<>(); @Inject public InternalAwsS3Service(Settings settings) { @@ -72,8 +62,12 @@ public InternalAwsS3Service(Settings settings) { } @Override -<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) { + return client(endpoint, protocol, region, account, key, maxRetries, null); + } + + @Override + public AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { if (Strings.isNullOrEmpty(endpoint)) { // We need to set the endpoint based on the region if (region != null) { @@ -83,50 +77,16 @@ public synchronized AmazonS3 client(String endpoint, Protocol protocol, String r // No region has been set so we will use the default endpoint endpoint = getDefaultEndpoint(); } -======= - public synchronized AmazonS3 client() { - String endpoint = getDefaultEndpoint(); - String account = settings.get("cloud.aws.access_key", settings.get("cloud.account")); - String key = settings.get("cloud.aws.secret_key", settings.get("cloud.key")); - - return getClient(endpoint, null, account, key, null, null); - } - - @Override - public AmazonS3 client(String endpoint, String protocol, String region, String account, String key) { - return client(endpoint, protocol, region, account, key, null); - } - - @Override - public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) { - return client(endpoint, protocol, region, account, key, maxRetries, null); - } - - @Override - public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { - if (region != null && endpoint == null) { - endpoint = getEndpoint(region); - logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); - } else if (endpoint == null) { - endpoint = getDefaultEndpoint(); - } - if (account == null || key == null) { - account = settings.get("cloud.aws.access_key", settings.get("cloud.account")); - key = settings.get("cloud.aws.secret_key", settings.get("cloud.key")); ->>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java } return getClient(endpoint, protocol, account, key, maxRetries, clientSideEncryptionMaterials); } -<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java - private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) { - Tuple clientDescriptor = new Tuple<>(endpoint, account); -======= + private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries, + EncryptionMaterials clientSideEncryptionMaterials) { - private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { - Tuple> clientDescriptor = new Tuple>(endpoint, new Tuple(account, clientSideEncryptionMaterials)); ->>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java + Tuple tempTuple = new Tuple<>(account, clientSideEncryptionMaterials); + Tuple> clientDescriptor = new Tuple<>(endpoint, tempTuple); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { return client; @@ -177,7 +137,7 @@ private synchronized AmazonS3 getClient(String endpoint, String protocol, String ); } - if(clientSideEncryptionMaterials != null) { + if (clientSideEncryptionMaterials != null) { EncryptionMaterialsProvider encryptionMaterialsProvider = new StaticEncryptionMaterialsProvider(clientSideEncryptionMaterials); CryptoConfiguration cryptoConfiguration = new CryptoConfiguration(); client = new AmazonS3EncryptionClient( diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java index dd278a9231d02..60f0b46322328 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java @@ -20,6 +20,8 @@ package org.elasticsearch.cloud.aws.blobstore; import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; @@ -131,31 +133,10 @@ protected void doUpload(S3BlobStore blobStore, String bucketName, String blobNam } md.setContentLength(length); - InputStream inputStream = is; - - // We try to compute a MD5 while reading it - MessageDigest messageDigest; - try { - messageDigest = MessageDigest.getInstance("MD5"); - inputStream = new DigestInputStream(is, messageDigest); - } catch (NoSuchAlgorithmException impossible) { - // Every implementation of the Java platform is required to support MD5 (see MessageDigest) - throw new RuntimeException(impossible); - } - - PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, inputStream, md) + PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, is, md) .withStorageClass(blobStore.getStorageClass()) .withCannedAcl(blobStore.getCannedACL()); - PutObjectResult putObjectResult = blobStore.client().putObject(putRequest); - - String localMd5 = Base64.encodeAsString(messageDigest.digest()); - String remoteMd5 = putObjectResult.getContentMd5(); - if (!localMd5.equals(remoteMd5)) { - logger.debug("MD5 local [{}], remote [{}] are not equal...", localMd5, remoteMd5); - throw new AmazonS3Exception("MD5 local [" + localMd5 + - "], remote [" + remoteMd5 + - "] are not equal..."); - } + blobStore.client().putObject(putRequest); } private void initializeMultipart() { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index 82eb9f63ee293..f2e70919e73e4 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -21,13 +21,10 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; import com.amazonaws.services.s3.model.AmazonS3Exception; -<<<<<<< HEAD:plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.CreateBucketRequest; -======= -import com.amazonaws.services.s3.AmazonS3EncryptionClient; ->>>>>>> 98d508f... Add client-side encryption:src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.ObjectListing; @@ -79,7 +76,7 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable throw new BlobStoreException("Detected client-side encryption " + "and a buffer_size for the S3 storage not a multiple of the cipher block size (16)"); } - + this.cannedACL = initCannedACL(cannedACL); this.numberOfRetries = maxRetries; this.storageClass = initStorageClass(storageClass); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java index d07d8c174c514..99f57547a5cb0 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java @@ -128,6 +128,9 @@ public void onModule(SettingsModule settingsModule) { settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING); settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING); settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING); + settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PRIVATE_KEY); + settingsModule.registerSetting(S3Repository.Repositories.CLIENT_PUBLIC_KEY); + settingsModule.registerSetting(S3Repository.Repositories.CLIENT_SYMMETRIC_KEY); // Register S3 single repository settings settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING); @@ -144,6 +147,10 @@ public void onModule(SettingsModule settingsModule) { settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING); settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING); settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING); + + settingsModule.registerSetting(S3Repository.Repository.CLIENT_PRIVATE_KEY); + settingsModule.registerSetting(S3Repository.Repository.CLIENT_PUBLIC_KEY); + settingsModule.registerSetting(S3Repository.Repository.CLIENT_SYMMETRIC_KEY); } /** diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index a09d57ebc93d3..4549cc5214ce6 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -20,6 +20,8 @@ package org.elasticsearch.repositories.s3; import com.amazonaws.Protocol; +import com.amazonaws.services.s3.model.EncryptionMaterials; +import com.amazonaws.util.Base64; import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3; import org.elasticsearch.cloud.aws.blobstore.S3BlobStore; @@ -37,7 +39,15 @@ import org.elasticsearch.repositories.RepositorySettings; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import javax.crypto.Cipher; +import javax.crypto.spec.SecretKeySpec; import java.io.IOException; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; import java.util.Locale; import java.util.function.Function; @@ -140,6 +150,21 @@ public interface Repositories { * repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory. */ Setting BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope); + /** + * repositories.s3.client_symmetric_key: Specifies the Base64-encoded AES symmetric-key (128, 192 or 256 bits) + */ + Setting CLIENT_SYMMETRIC_KEY = Setting.simpleString("repositories.s3.client_symmetric_key", Property.NodeScope); + + /** + * repositories.s3.client_public_key: Specifies the Base64-encoded RSA public key + */ + Setting CLIENT_PUBLIC_KEY = Setting.simpleString("repositories.s3.client_public_key", Property.NodeScope); + + /** + * repositories.s3.client_private_key: Specifies the Base64-encoded RSA private key + */ + Setting CLIENT_PRIVATE_KEY = Setting.simpleString("repositories.s3.client_private_key", Property.NodeScope); + } /** @@ -222,7 +247,26 @@ public interface Repository { * base_path * @see Repositories#BASE_PATH_SETTING */ + Setting BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope); + + /** + * base_path + * @see Repositories#CLIENT_SYMMETRIC_KEY + */ + Setting CLIENT_SYMMETRIC_KEY = Setting.simpleString("client_symmetric_key", Property.NodeScope); + + /** + * base_path + * @see Repositories#CLIENT_PUBLIC_KEY + */ + Setting CLIENT_PUBLIC_KEY = Setting.simpleString("client_public_key", Property.NodeScope); + + /** + * base_path + * @see Repositories#CLIENT_PRIVATE_KEY + */ + Setting CLIENT_PRIVATE_KEY = Setting.simpleString("client_private_key", Property.NodeScope); } private final S3BlobStore blobStore; @@ -275,14 +319,30 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING); String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING); - logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]", + logger.debug("using bqucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]", bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass); String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING); String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING); - blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries), - bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass); + // parse and validate the client side encryption setting + String symmetricKeyBase64 = getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY); + String publicKeyBase64 =getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY); + String privateKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY); + + EncryptionMaterials clientSideEncryptionMaterials = initClientSideEncryption(symmetricKeyBase64, publicKeyBase64, privateKeyBase64, name); + + blobStore = new S3BlobStore( + settings, + s3Service.client(endpoint, protocol, region, key, secret, maxRetries, clientSideEncryptionMaterials), + bucket, + region, + serverSideEncryption, + bufferSize, + maxRetries, + cannedACL, + storageClass + ); String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING); if (Strings.hasLength(basePath)) { @@ -294,6 +354,52 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, } else { this.basePath = BlobPath.cleanPath(); } + + } + + + /** + * Init and verify initClientSideEncryption settings + */ + private EncryptionMaterials initClientSideEncryption(String symmetricKey, String publicKey, String privateKey, RepositoryName name) { + + EncryptionMaterials clientSideEncryptionMaterials = null; + + if (Strings.isNullOrEmpty(symmetricKey) == false && (Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false)) { + throw new RepositoryException(name.name(), "Client-side encryption: You can't specify an symmetric key AND a public/private key pair"); + } + + if (Strings.isNullOrEmpty(symmetricKey) == false || Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false) { + try { + // Check crypto + if (Cipher.getMaxAllowedKeyLength("AES") < 256) { + throw new RepositoryException(name.name(), "Client-side encryption: Please install the Java Cryptography Extension"); + } + + // Transform the keys in a EncryptionMaterials + if (Strings.isNullOrEmpty(symmetricKey) == false) { + clientSideEncryptionMaterials = new EncryptionMaterials(new SecretKeySpec(Base64.decode(symmetricKey), "AES")); + } else { + if (Strings.isNullOrEmpty(publicKey)|| Strings.isNullOrEmpty(privateKey)){ + String missingKey = Strings.isNullOrEmpty(publicKey) ? "public key" : "private key"; + throw new RepositoryException(name.name(), "Client-side encryption: " + missingKey + " is missing"); + } + + clientSideEncryptionMaterials = new EncryptionMaterials(new KeyPair( + KeyFactory.getInstance("RSA").generatePublic(new X509EncodedKeySpec(Base64.decode(publicKey))), + KeyFactory.getInstance("RSA").generatePrivate(new PKCS8EncodedKeySpec(Base64.decode(privateKey))))); + } + + } catch (IllegalArgumentException e) { + throw new RepositoryException(name.name(), "Client-side encryption: Error decoding your keys: " + e.getMessage()); + } catch (NoSuchAlgorithmException e) { + throw new RepositoryException(name.name(), e.getMessage()); + } catch (InvalidKeySpecException e) { + throw new RepositoryException(name.name(), e.getMessage()); + } + } + + return clientSideEncryptionMaterials; } /** diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index e68a06c1ddfe1..578af39bfe283 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -23,15 +23,9 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; -<<<<<<< HEAD:plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java -======= import com.amazonaws.util.Base64; -import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.stream.JsonReader; -import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.stream.MalformedJsonException; ->>>>>>> 98d508f... Add client-side encryption:src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -41,11 +35,7 @@ import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; -<<<<<<< HEAD:plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java -======= -import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoryException; ->>>>>>> 98d508f... Add client-side encryption:src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryVerificationException; import org.elasticsearch.snapshots.SnapshotMissingException; @@ -56,8 +46,10 @@ import org.junit.Before; import javax.crypto.KeyGenerator; -import java.io.InputStreamReader; -import java.security.*; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -112,7 +104,7 @@ public void testSimpleWorkflow() { logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(settings - ).get(); + ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); @@ -135,7 +127,6 @@ public void testSimpleWorkflow() { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - assertMetadataFileIsNotEncrypted("test-snap"); logger.info("--> delete some data"); for (int i = 0; i < 50; i++) { @@ -267,77 +258,6 @@ public void testEncryption() { assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") - public void testClientSideEncryption() throws NoSuchAlgorithmException { - - KeyGenerator keyGenerator1 = KeyGenerator.getInstance("AES"); - keyGenerator1.init(128); - String symmetricEncryptionKeyBase64 = Base64.encodeAsString(keyGenerator1.generateKey().getEncoded()); - - KeyPairGenerator keyGenerator2 = KeyPairGenerator.getInstance("RSA"); - keyGenerator2.initialize(512, new SecureRandom()); - KeyPair keyPair = keyGenerator2.generateKeyPair(); - String publicEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPublic().getEncoded()); - String privateEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPrivate().getEncoded()); - - Client client = client(); - try { - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(ImmutableSettings.settingsBuilder() - .put("base_path", basePath) - .put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64) - .put("client_side_encryption_key.public", publicEncryptionKeyBase64) - .put("client_side_encryption_key.private", privateEncryptionKeyBase64) - .put("chunk_size", randomIntBetween(1000, 10000)) - ).get(); - fail("Symmetric and public/private key pairs are exclusive options. An exception should be thrown."); - } catch(RepositoryException e) { - } - - List allSettings = Arrays.asList( - ImmutableSettings.settingsBuilder() - .put("base_path", basePath) - .put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64) - .put("chunk_size", randomIntBetween(1000, 10000)), - ImmutableSettings.settingsBuilder() - .put("base_path", basePath) - .put("client_side_encryption_key.public", publicEncryptionKeyBase64) - .put("client_side_encryption_key.private", privateEncryptionKeyBase64) - .put("chunk_size", randomIntBetween(1000, 10000)) - ); - for(ImmutableSettings.Builder settings: allSettings) { - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(settings).get(); - - // Create the index and index some data - createIndex("test-idx-1"); - for (int i = 0; i < 100; i++) { - index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - - // Take the snapshot - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - - assertMetadataFileIsEncrypted("test-snap"); - - // Restore - cluster().wipeIndices("test-idx-1"); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet(); - ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); - assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); - - // Clean, the test will bbe run with different settings - cluster().wipeIndices("test-idx-1"); - wipeRepositories(); - cleanRepositoryFiles(basePath); - } - } - /** * This test verifies that the test configuration is set up in a manner that * does not make the test {@link #testRepositoryWithCustomCredentials()} pointless. @@ -488,6 +408,75 @@ public void testGetDeleteNonExistingSnapshot86() { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") + public void testClientSideEncryption() throws NoSuchAlgorithmException { + + KeyGenerator keyGenerator1 = KeyGenerator.getInstance("AES"); + keyGenerator1.init(128); + String symmetricEncryptionKeyBase64 = Base64.encodeAsString(keyGenerator1.generateKey().getEncoded()); + + KeyPairGenerator keyGenerator2 = KeyPairGenerator.getInstance("RSA"); + keyGenerator2.initialize(512, new SecureRandom()); + KeyPair keyPair = keyGenerator2.generateKeyPair(); + String publicEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPublic().getEncoded()); + String privateEncryptionKeyBase64 = Base64.encodeAsString(keyPair.getPrivate().getEncoded()); + + Client client = client(); + try { + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(Settings.settingsBuilder() + .put("base_path", basePath) + .put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64) + .put("client_side_encryption_key.public", publicEncryptionKeyBase64) + .put("client_side_encryption_key.private", privateEncryptionKeyBase64) + .put("chunk_size", randomIntBetween(1000, 10000)) + ).get(); + fail("Symmetric and public/private key pairs are exclusive options. An exception should be thrown."); + } catch (RepositoryException e) { + } + + List allSettings = Arrays.asList( + Settings.settingsBuilder() + .put("base_path", basePath) + .put("client_side_encryption_key.symmetric", symmetricEncryptionKeyBase64) + .put("chunk_size", randomIntBetween(1000, 10000)), + Settings.settingsBuilder() + .put("base_path", basePath) + .put("client_side_encryption_key.public", publicEncryptionKeyBase64) + .put("client_side_encryption_key.private", privateEncryptionKeyBase64) + .put("chunk_size", randomIntBetween(1000, 10000)) + ); + for (Settings.Builder settings : allSettings) { + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(settings).get(); + + // Create the index and index some data + createIndex("test-idx-1"); + for (int i = 0; i < 100; i++) { + index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + // Take the snapshot + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + // Restore + cluster().wipeIndices("test-idx-1"); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet(); + ensureGreen(); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); + assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); + + // Clean, the test will bbe run with different settings + cluster().wipeIndices("test-idx-1"); + wipeRepositories(); + cleanRepositoryFiles(basePath); + } + } + private void assertRepositoryIsOperational(Client client, String repository) { createIndex("test-idx-1"); ensureGreen(); @@ -524,51 +513,6 @@ private void assertRepositoryIsOperational(Client client, String repository) { assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); } - private void assertMetadataFileIsEncrypted(String snapshotName) { - - Settings settings = internalCluster().getInstance(Settings.class); - AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( - settings.get("repositories.s3.endpoint"), - settings.get("repositories.s3.protocol"), - settings.get("repositories.s3.region"), - settings.get("cloud.aws.access_key"), - settings.get("cloud.aws.secret_key")); - String bucket = settings.get("repositories.s3.bucket"); - String objectKey = basePath + "/metadata-" + snapshotName; - S3Object object = s3Client.getObject(bucket, objectKey); - - try { - JsonReader jsonReader = new JsonReader(new InputStreamReader(object.getObjectContent())); - jsonReader.beginObject(); - assertThat("The file hasn't been encrypted properly, its content is still readable!", jsonReader.nextName(), not(equalTo("meta-data"))); - } catch(Exception e) { - // The json is not valid, the file is encrypted - - // MalformedJsonException can't be catched directly so the following - // assertion is necessary to avoid silent failures. - assertThat(e, instanceOf(MalformedJsonException.class)); - } - } - - private void assertMetadataFileIsNotEncrypted(String snapshotName) { - - Settings settings = internalCluster().getInstance(Settings.class); - AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( - settings.get("repositories.s3.endpoint"), - settings.get("repositories.s3.protocol"), - settings.get("repositories.s3.region"), - settings.get("cloud.aws.access_key"), - settings.get("cloud.aws.secret_key")); - String bucket = settings.get("repositories.s3.bucket"); - String objectKey = basePath + "/metadata-" + snapshotName; - S3Object object = s3Client.getObject(bucket, objectKey); - - JsonReader jsonReader = new JsonReader(new InputStreamReader(object.getObjectContent())); - jsonReader.beginObject(); - assertThat("The file wasn't decrypted properly", jsonReader.nextName(), equalTo("meta-data")); - - // The beginning of the file looks like json. If it was encrypted, it wouldn't. - } /** * Deletes repositories, supports wildcard notation. From 38923b89c278ddc6c790655f5bd591f9ed7ec233 Mon Sep 17 00:00:00 2001 From: Xu Zhang Date: Sun, 6 Mar 2016 21:05:52 -0800 Subject: [PATCH 3/3] Update Format, add new settings into the setting test --- .../java/org/elasticsearch/cloud/aws/AwsS3Service.java | 1 - .../elasticsearch/cloud/aws/InternalAwsS3Service.java | 10 +++------- .../cloud/aws/blobstore/DefaultS3OutputStream.java | 2 -- .../plugin/repository/s3/S3RepositoryPlugin.java | 1 - .../elasticsearch/repositories/s3/S3Repository.java | 9 ++++----- .../cloud/aws/RepositoryS3SettingsTests.java | 8 ++++++++ .../org/elasticsearch/cloud/aws/TestAwsS3Service.java | 5 +++-- .../repositories/s3/AbstractS3SnapshotRestoreTest.java | 5 +++-- 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index 4b98176a14a77..1fc7e5decd4a2 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -154,6 +154,5 @@ interface CLOUD_S3 { Setting ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope); } - AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries); AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 30a48dddc14f5..4bae81ee74de3 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -62,12 +62,9 @@ public InternalAwsS3Service(Settings settings) { } @Override - public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) { - return client(endpoint, protocol, region, account, key, maxRetries, null); - } + public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, + EncryptionMaterials clientSideEncryptionMaterials) { - @Override - public AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { if (Strings.isNullOrEmpty(endpoint)) { // We need to set the endpoint based on the region if (region != null) { @@ -144,8 +141,7 @@ private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, Stri credentials, encryptionMaterialsProvider, clientConfiguration, - cryptoConfiguration - ); + cryptoConfiguration); } else { client = new AmazonS3Client(credentials, clientConfiguration); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java index 60f0b46322328..2fa6400b03fb1 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java @@ -20,8 +20,6 @@ package org.elasticsearch.cloud.aws.blobstore; import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3EncryptionClient; -import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java index 99f57547a5cb0..7b12777e4f983 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java @@ -147,7 +147,6 @@ public void onModule(SettingsModule settingsModule) { settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING); settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING); settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING); - settingsModule.registerSetting(S3Repository.Repository.CLIENT_PRIVATE_KEY); settingsModule.registerSetting(S3Repository.Repository.CLIENT_PUBLIC_KEY); settingsModule.registerSetting(S3Repository.Repository.CLIENT_SYMMETRIC_KEY); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 4549cc5214ce6..d6f830168f65d 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -247,7 +247,6 @@ public interface Repository { * base_path * @see Repositories#BASE_PATH_SETTING */ - Setting BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope); /** @@ -319,7 +318,7 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING); String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING); - logger.debug("using bqucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]", + logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]", bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass); String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING); @@ -327,7 +326,7 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, // parse and validate the client side encryption setting String symmetricKeyBase64 = getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY); - String publicKeyBase64 =getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY); + String publicKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY); String privateKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY); EncryptionMaterials clientSideEncryptionMaterials = initClientSideEncryption(symmetricKeyBase64, publicKeyBase64, privateKeyBase64, name); @@ -366,7 +365,7 @@ private EncryptionMaterials initClientSideEncryption(String symmetricKey, String EncryptionMaterials clientSideEncryptionMaterials = null; if (Strings.isNullOrEmpty(symmetricKey) == false && (Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false)) { - throw new RepositoryException(name.name(), "Client-side encryption: You can't specify an symmetric key AND a public/private key pair"); + throw new RepositoryException(name.name(), "Client-side encryption: You can't specify a symmetric key AND a public/private key pair"); } if (Strings.isNullOrEmpty(symmetricKey) == false || Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false) { @@ -380,7 +379,7 @@ private EncryptionMaterials initClientSideEncryption(String symmetricKey, String if (Strings.isNullOrEmpty(symmetricKey) == false) { clientSideEncryptionMaterials = new EncryptionMaterials(new SecretKeySpec(Base64.decode(symmetricKey), "AES")); } else { - if (Strings.isNullOrEmpty(publicKey)|| Strings.isNullOrEmpty(privateKey)){ + if (Strings.isNullOrEmpty(publicKey) || Strings.isNullOrEmpty(privateKey)) { String missingKey = Strings.isNullOrEmpty(publicKey) ? "public key" : "private key"; throw new RepositoryException(name.name(), "Client-side encryption: " + missingKey + " is missing"); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java index c5c617e8591f5..50ec7b02cddbf 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java @@ -97,6 +97,7 @@ public class RepositoryS3SettingsTests extends ESTestCase { .put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class") .put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl") .put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath") + .build(); /** @@ -125,6 +126,9 @@ public void testRepositorySettingsGlobalOnly() { assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString()); + assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString()); + assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString()); + assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString()); } /** @@ -153,6 +157,9 @@ public void testRepositorySettingsGlobalOverloadedByS3() { assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString()); + assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString()); + assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString()); + assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString()); } /** @@ -329,6 +336,7 @@ private Settings buildSettings(Settings... global) { private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage) throws IOException { + Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES); RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder() .put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java index 47e884d73bd46..46c42b0a1cebe 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -20,6 +20,7 @@ import com.amazonaws.Protocol; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.EncryptionMaterials; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -51,8 +52,8 @@ public TestAwsS3Service(Settings settings) { @Override - public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) { - return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries)); + public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) { + return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, clientSideEncryptionMaterials)); } private AmazonS3 cachedWrapper(AmazonS3 client) { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 578af39bfe283..d47978e71bb43 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -104,7 +104,7 @@ public void testSimpleWorkflow() { logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(settings - ).get(); + ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); @@ -209,6 +209,7 @@ public void testEncryption() { S3Repository.Repositories.REGION_SETTING.get(settings), S3Repository.Repositories.KEY_SETTING.get(settings), S3Repository.Repositories.SECRET_SETTING.get(settings), + null, null); String bucketName = bucket.get("bucket"); @@ -553,7 +554,7 @@ public void cleanRepositoryFiles(String basePath) { // We check that settings has been set in elasticsearch.yml integration test file // as described in README assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); - AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null); + AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null, null); try { ObjectListing prevListing = null; //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html