From fb0dfb4ec70e79358a2034f8ccd7357159086aa0 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 17:51:52 +0100 Subject: [PATCH 01/86] Update to elasticsearch 1.1.0 / Lucene 4.7.0 Closes #60. --- pom.xml | 6 +++--- .../java/org/elasticsearch/discovery/ec2/Ec2Discovery.java | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 176cb52c..16a4de68 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.0.0-SNAPSHOT + 2.1.0-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. @@ -32,8 +32,8 @@ - 1.0.0 - 4.6.1 + 1.1.0 + 4.7.0 onerror 1 true diff --git a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index 3a508019..5d85558b 100755 --- a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingService; @@ -43,8 +44,10 @@ public class Ec2Discovery extends ZenDiscovery { @Inject public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, - DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service) { - super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService, Version.CURRENT); + DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, + DiscoverySettings discoverySettings) { + super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, + discoveryNodeService, pingService, Version.CURRENT, discoverySettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From e9f3a395996ec79b31454d4e1652607ba2d2ef12 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 18:32:08 +0100 Subject: [PATCH 02/86] Create branches according to elasticsearch versions We create branches: * es-0.90 for elasticsearch 0.90 * es-1.0 for elasticsearch 1.0 * es-1.1 for elasticsearch 1.1 * master for elasticsearch master We also check that before releasing we don't have a dependency to an elasticsearch SNAPSHOT version. Add links to each version in documentation (cherry picked from commit e0f06c6) --- README.md | 13 +++++++++---- dev-tools/build_release.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 888f8b44..a8b738fb 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,18 @@ for the unicast discovery mechanism and add S3 repositories. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.0.0.RC1`. -* For 1.0.x elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). -* For 0.90.x elasticsearch versions, look at [1.x branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/1.x). +* For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). +* For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.1). +* For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.0). +* For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-0.90). | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| -| 2.0.0-SNAPSHOT | 1.0.0.RC1 -> master | XXXX-XX-XX | -| 2.0.0.RC1 | 1.0.0.RC1 -> master | 2014-01-15 | +| 2.1.0-SNAPSHOT | 1.1.0 -> 1.1 | XXXX-XX-XX | + +Please read documentation relative to the version you are using: + +* [2.1.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.1/README.md) ## Generic Configuration diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index 74acd8c5..db834544 100755 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -208,6 +208,29 @@ def callback(line): return line process_file(readme_file, callback) +# Moves the README.md file from a snapshot to a release (documentation link) +def remove_documentation_snapshot(readme_file, repo_url, release, branch): + pattern = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (release, repo_url, branch) + replacement = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) + def callback(line): + # If we find pattern, we replace its content + if line.find(pattern) >= 0: + return line.replace(pattern, replacement) + else: + return line + process_file(readme_file, callback) + +# Add in README.markdown file the documentation for the next version +def add_documentation_snapshot(readme_file, repo_url, release, snapshot, branch): + pattern = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) + replacement = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (snapshot, repo_url, branch) + def callback(line): + # If we find pattern, we copy the line and replace its content + if line.find(pattern) >= 0: + return line.replace(pattern, replacement)+line + else: + return line + process_file(readme_file, callback) # Set release date in README.md file def set_date(readme_file): @@ -603,8 +626,12 @@ def check_email_settings(): artifact_name = find_from_pom('name') artifact_description = find_from_pom('description') project_url = find_from_pom('url') + elasticsearch_version = find_from_pom('elasticsearch.version') print(' Artifact Id: [%s]' % artifact_id) print(' Release version: [%s]' % release_version) + print(' Elasticsearch: [%s]' % elasticsearch_version) + if elasticsearch_version.find('-SNAPSHOT') != -1: + raise RuntimeError('Can not release with a SNAPSHOT elasticsearch dependency: %s' % elasticsearch_version) # extract snapshot default_snapshot_version = guess_snapshot(release_version) @@ -626,6 +653,7 @@ def check_email_settings(): try: pending_files = [POM_FILE, README_FILE] remove_maven_snapshot(POM_FILE, release_version) + remove_documentation_snapshot(README_FILE, project_url, release_version, src_branch) remove_version_snapshot(README_FILE, release_version) set_date(README_FILE) set_install_instructions(README_FILE, artifact_id, release_version) @@ -657,6 +685,7 @@ def check_email_settings(): add_maven_snapshot(POM_FILE, release_version, snapshot_version) add_version_snapshot(README_FILE, release_version, snapshot_version) + add_documentation_snapshot(README_FILE, project_url, release_version, snapshot_version, src_branch) add_pending_files(*pending_files) commit_snapshot() From ef754eb5964ded325648b0af3e7dd9e04a1aa304 Mon Sep 17 00:00:00 2001 From: Konrad Beiske Date: Wed, 26 Mar 2014 18:51:03 +0100 Subject: [PATCH 03/86] Add per repository credentials Changed AwsS3Service to use one client per region and credentials combination. Made S3Repository specify credentials if such exists in the repository settings. Updated readme with repository specific credentials settings. Closes #54. Closes #55. Closes #56. (cherry picked from commit d4ea2dd) --- README.md | 15 +- .../elasticsearch/cloud/aws/AwsS3Service.java | 127 ++++++++---- .../repositories/s3/S3Repository.java | 2 +- .../s3/S3SnapshotRestoreTest.java | 193 ++++++++++++++---- src/test/resources/elasticsearch.yml | 19 +- 5 files changed, 269 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index a8b738fb..06bd352b 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,8 @@ The following settings are supported: * `bucket`: The name of the bucket to be used for snapshots. (Mandatory) * `region`: The region where bucket is located. Defaults to US Standard * `base_path`: Specifies the path within bucket to repository data. Defaults to root directory. +* `access_key`: The access key to use for authentication. Defaults to value of `cloud.aws.access_key`. +* `secret_key`: The secret key to use for authentication. Defaults to value of `cloud.aws.secret_key`. * `concurrent_streams`: Throttles the number of streams (per node) preforming snapshot operation. Defaults to `5`. * `chunk_size`: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, i.e. `1g`, `10m`, `5k`. Defaults to `100m`. * `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`. @@ -131,11 +133,11 @@ The S3 repositories are using the same credentials as the rest of the S3 service secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br -Multiple S3 repositories can be created as long as they share the same credential. +Multiple S3 repositories can be created. If the buckets require different credentials, then define them as part of the repository settings. ## Testing -Integrations tests in this plugin require working AWS configuration and therefore disabled by default. To enable tests prepare a config file elasticsearch.yml with the following content: +Integrations tests in this plugin require working AWS configuration and therefore disabled by default. Three buckets and two iam users have to be created. The first iam user needs access to two buckets in different regions and the final bucket is exclusive for the other iam user. To enable tests prepare a config file elasticsearch.yml with the following content: ``` cloud: @@ -147,10 +149,17 @@ repositories: s3: bucket: "bucket_name" region: "us-west-2" + private-bucket: + bucket: + access_key: + secret_key: + remote-bucket: + bucket: + region: ``` -Replaces `access_key`, `secret_key`, `bucket` and `region` with your settings. Please, note that the test will delete all snapshot/restore related files in the specified bucket. +Replace all occurrences of `access_key`, `secret_key`, `bucket` and `region` with your settings. Please, note that the test will delete all snapshot/restore related files in the specified buckets. To run test: diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index 77c5c588..80127db4 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -19,6 +19,9 @@ package org.elasticsearch.cloud.aws; +import java.util.HashMap; +import java.util.Map; + import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.*; @@ -27,6 +30,7 @@ import com.amazonaws.services.s3.AmazonS3Client; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -37,7 +41,10 @@ */ public class AwsS3Service extends AbstractLifecycleComponent { - private AmazonS3Client client; + /** + * (acceskey, endpoint) -> client + */ + private Map, AmazonS3Client> clients = new HashMap, AmazonS3Client>(); @Inject public AwsS3Service(Settings settings, SettingsFilter settingsFilter) { @@ -47,6 +54,33 @@ public AwsS3Service(Settings settings, SettingsFilter settingsFilter) { } public synchronized AmazonS3 client() { + String endpoint = getDefaultEndpoint(); + String account = componentSettings.get("access_key", settings.get("cloud.account")); + String key = componentSettings.get("secret_key", settings.get("cloud.key")); + + return getClient(endpoint, account, key); + } + + public synchronized AmazonS3 client(String region, String account, String key) { + String endpoint; + if (region == null) { + endpoint = getDefaultEndpoint(); + } else { + endpoint = getEndpoint(region); + logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + } + if (account == null || key == null) { + account = componentSettings.get("access_key", settings.get("cloud.account")); + key = componentSettings.get("secret_key", settings.get("cloud.key")); + } + + return getClient(endpoint, account, key); + } + + + private synchronized AmazonS3 getClient(String endpoint, String account, String key) { + Tuple clientDescriptor = new Tuple(endpoint, account); + AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { return client; } @@ -60,8 +94,6 @@ public synchronized AmazonS3 client() { } else { throw new ElasticsearchIllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]"); } - String account = componentSettings.get("access_key", settings.get("cloud.account")); - String key = componentSettings.get("secret_key", settings.get("cloud.key")); String proxyHost = componentSettings.get("proxy_host"); if (proxyHost != null) { @@ -88,53 +120,60 @@ public synchronized AmazonS3 client() { new StaticCredentialsProvider(new BasicAWSCredentials(account, key)) ); } - this.client = new AmazonS3Client(credentials, clientConfiguration); + client = new AmazonS3Client(credentials, clientConfiguration); + if (endpoint != null) { + client.setEndpoint(endpoint); + } + clients.put(clientDescriptor, client); + return client; + } + + private String getDefaultEndpoint() { + String endpoint = null; if (componentSettings.get("s3.endpoint") != null) { - String endpoint = componentSettings.get("s3.endpoint"); + endpoint = componentSettings.get("s3.endpoint"); logger.debug("using explicit s3 endpoint [{}]", endpoint); - client.setEndpoint(endpoint); } else if (componentSettings.get("region") != null) { - String endpoint; String region = componentSettings.get("region").toLowerCase(); - if ("us-east".equals(region)) { - endpoint = "s3.amazonaws.com"; - } else if ("us-east-1".equals(region)) { - endpoint = "s3.amazonaws.com"; - } else if ("us-west".equals(region)) { - endpoint = "s3-us-west-1.amazonaws.com"; - } else if ("us-west-1".equals(region)) { - endpoint = "s3-us-west-1.amazonaws.com"; - } else if ("us-west-2".equals(region)) { - endpoint = "s3-us-west-2.amazonaws.com"; - } else if ("ap-southeast".equals(region)) { - endpoint = "s3-ap-southeast-1.amazonaws.com"; - } else if ("ap-southeast-1".equals(region)) { - endpoint = "s3-ap-southeast-1.amazonaws.com"; - } else if ("ap-southeast-2".equals(region)) { - endpoint = "s3-ap-southeast-2.amazonaws.com"; - } else if ("ap-northeast".equals(region)) { - endpoint = "s3-ap-northeast-1.amazonaws.com"; - } else if ("ap-northeast-1".equals(region)) { - endpoint = "s3-ap-northeast-1.amazonaws.com"; - } else if ("eu-west".equals(region)) { - endpoint = "s3-eu-west-1.amazonaws.com"; - } else if ("eu-west-1".equals(region)) { - endpoint = "s3-eu-west-1.amazonaws.com"; - } else if ("sa-east".equals(region)) { - endpoint = "s3-sa-east-1.amazonaws.com"; - } else if ("sa-east-1".equals(region)) { - endpoint = "s3-sa-east-1.amazonaws.com"; - } else { - throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); - } - if (endpoint != null) { - logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); - client.setEndpoint(endpoint); - } + endpoint = getEndpoint(region); + logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); } + return endpoint; + } - return this.client; + private static String getEndpoint(String region) { + if ("us-east".equals(region)) { + return "s3.amazonaws.com"; + } else if ("us-east-1".equals(region)) { + return "s3.amazonaws.com"; + } else if ("us-west".equals(region)) { + return "s3-us-west-1.amazonaws.com"; + } else if ("us-west-1".equals(region)) { + return "s3-us-west-1.amazonaws.com"; + } else if ("us-west-2".equals(region)) { + return "s3-us-west-2.amazonaws.com"; + } else if ("ap-southeast".equals(region)) { + return "s3-ap-southeast-1.amazonaws.com"; + } else if ("ap-southeast-1".equals(region)) { + return "s3-ap-southeast-1.amazonaws.com"; + } else if ("ap-southeast-2".equals(region)) { + return "s3-ap-southeast-2.amazonaws.com"; + } else if ("ap-northeast".equals(region)) { + return "s3-ap-northeast-1.amazonaws.com"; + } else if ("ap-northeast-1".equals(region)) { + return "s3-ap-northeast-1.amazonaws.com"; + } else if ("eu-west".equals(region)) { + return "s3-eu-west-1.amazonaws.com"; + } else if ("eu-west-1".equals(region)) { + return "s3-eu-west-1.amazonaws.com"; + } else if ("sa-east".equals(region)) { + return "s3-sa-east-1.amazonaws.com"; + } else if ("sa-east-1".equals(region)) { + return "s3-sa-east-1.amazonaws.com"; + } else { + throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); + } } @Override @@ -147,7 +186,7 @@ protected void doStop() throws ElasticsearchException { @Override protected void doClose() throws ElasticsearchException { - if (client != null) { + for (AmazonS3Client client : clients.values()) { client.shutdown(); } } diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 3f47f589..b9595a3a 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -124,7 +124,7 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, ExecutorService concurrentStreamPool = EsExecutors.newScaling(1, concurrentStreams, 5, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(settings, "[s3_stream]")); logger.debug("using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}]", bucket, region, chunkSize, concurrentStreams); - blobStore = new S3BlobStore(settings, s3Service.client(), bucket, region, concurrentStreamPool); + blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, concurrentStreamPool); this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); String basePath = repositorySettings.settings().get("base_path", null); diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index c057149f..1484c88c 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -23,6 +23,7 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; + import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -33,6 +34,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; @@ -50,7 +52,7 @@ /** */ @AwsTest -@ClusterScope(scope = Scope.TEST, numNodes = 2) +@ClusterScope(scope = Scope.SUITE, numNodes = 2) public class S3SnapshotRestoreTest extends AbstractAwsTest { @Override @@ -151,6 +153,117 @@ public void testSimpleWorkflow() { assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); } + /** + * This test verifies that the test configuration is set up in a manner that + * does not make the test {@link #testRepositoryWithCustomCredentials()} pointless. + */ + @Test(expected = UncategorizedExecutionException.class) + public void assertRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredentials() { + Client client = client(); + Settings bucketSettings = cluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("bucket", bucketSettings.get("bucket")) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + assertRepositoryIsOperational(client, "test-repo"); + } + + @Test + public void testRepositoryWithCustomCredentials() { + Client client = client(); + Settings bucketSettings = cluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("access_key", bucketSettings.get("access_key")) + .put("secret_key", bucketSettings.get("secret_key")) + .put("bucket", bucketSettings.get("bucket")) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + assertRepositoryIsOperational(client, "test-repo"); + } + + /** + * This test verifies that the test configuration is set up in a manner that + * does not make the test {@link #testRepositoryInRemoteRegion()} pointless. + */ + @Test(expected = UncategorizedExecutionException.class) + public void assertRepositoryInRemoteRegionIsRemote() { + Client client = client(); + Settings bucketSettings = cluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket."); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("bucket", bucketSettings.get("bucket")) +// Below setting intentionally omitted to assert bucket is not available in default region. +// .put("region", privateBucketSettings.get("region")) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + assertRepositoryIsOperational(client, "test-repo"); + } + + @Test + public void testRepositoryInRemoteRegion() { + Client client = client(); + Settings settings = cluster().getInstance(Settings.class); + Settings bucketSettings = settings.getByPrefix("repositories.s3.remote-bucket."); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("bucket", bucketSettings.get("bucket")) + .put("region", bucketSettings.get("region")) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + assertRepositoryIsOperational(client, "test-repo"); + } + + private void assertRepositoryIsOperational(Client client, String repository) { + createIndex("test-idx-1"); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repository, "test-snap").setWaitForCompletion(true).setIndices("test-idx-*").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + assertThat(client.admin().cluster().prepareGetSnapshots(repository).setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + + logger.info("--> delete some data"); + for (int i = 0; i < 50; i++) { + client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); + } + refresh(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); + + logger.info("--> close indices"); + client.admin().indices().prepareClose("test-idx-1").get(); + + logger.info("--> restore all indices from the snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repository, "test-snap").setWaitForCompletion(true).execute().actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + ensureGreen(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + } + + /** * Deletes repositories, supports wildcard notation. */ @@ -172,45 +285,55 @@ public static void wipeRepositories(String... repositories) { * Deletes content of the repository files in the bucket */ public void cleanRepositoryFiles(String basePath) { - String bucket = cluster().getInstance(Settings.class).get("repositories.s3.bucket"); - AmazonS3 client = cluster().getInstance(AwsS3Service.class).client(); - try { - ObjectListing prevListing = null; - //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html - //we can do at most 1K objects per delete - //We don't know the bucket name until first object listing - DeleteObjectsRequest multiObjectDeleteRequest = null; - ArrayList keys = new ArrayList(); - while (true) { - ObjectListing list; - if (prevListing != null) { - list = client.listNextBatchOfObjects(prevListing); - } else { - list = client.listObjects(bucket, basePath); - multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); - } - for (S3ObjectSummary summary : list.getObjectSummaries()) { - keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); - //Every 500 objects batch the delete request - if (keys.size() > 500) { - multiObjectDeleteRequest.setKeys(keys); - client.deleteObjects(multiObjectDeleteRequest); + Settings settings = cluster().getInstance(Settings.class); + Settings[] buckets = { + settings.getByPrefix("repositories.s3."), + settings.getByPrefix("repositories.s3.private-bucket."), + settings.getByPrefix("repositories.s3.remote-bucket.") + }; + for (Settings bucket : buckets) { + AmazonS3 client = cluster().getInstance(AwsS3Service.class).client( + bucket.get("region", settings.get("repositories.s3.region")), + bucket.get("access_key", settings.get("cloud.aws.access_key")), + bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); + try { + ObjectListing prevListing = null; + //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html + //we can do at most 1K objects per delete + //We don't know the bucket name until first object listing + DeleteObjectsRequest multiObjectDeleteRequest = null; + ArrayList keys = new ArrayList(); + while (true) { + ObjectListing list; + if (prevListing != null) { + list = client.listNextBatchOfObjects(prevListing); + } else { + list = client.listObjects(bucket.get("bucket"), basePath); multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); - keys.clear(); + } + for (S3ObjectSummary summary : list.getObjectSummaries()) { + keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); + //Every 500 objects batch the delete request + if (keys.size() > 500) { + multiObjectDeleteRequest.setKeys(keys); + client.deleteObjects(multiObjectDeleteRequest); + multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); + keys.clear(); + } + } + if (list.isTruncated()) { + prevListing = list; + } else { + break; } } - if (list.isTruncated()) { - prevListing = list; - } else { - break; + if (!keys.isEmpty()) { + multiObjectDeleteRequest.setKeys(keys); + client.deleteObjects(multiObjectDeleteRequest); } + } catch (Throwable ex) { + logger.warn("Failed to delete S3 repository", ex); } - if (!keys.isEmpty()) { - multiObjectDeleteRequest.setKeys(keys); - client.deleteObjects(multiObjectDeleteRequest); - } - } catch (Throwable ex) { - logger.warn("Failed to delete S3 repository", ex); } } } diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml index cf2dfeed..a7200fa3 100644 --- a/src/test/resources/elasticsearch.yml +++ b/src/test/resources/elasticsearch.yml @@ -1,10 +1,21 @@ # Replace this access_key / secret_key and bucket name with your own if you want # to run tests. -#cloud: +# cloud: # aws: -# access_key: AKVAIQBF2RECL7FJWGJQ -# secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br +# access_key: +# secret_key: # #discovery: # type: ec2 - +# +#repositories: +# s3: +# bucket: +# region: +# private-bucket: +# bucket: +# access_key: +# secret_key: +# remote-bucket: +# bucket: +# region: \ No newline at end of file From 18de58952cc78abe1ecc4e1e9f337adbb464cc18 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 19:05:29 +0100 Subject: [PATCH 04/86] prepare release elasticsearch-cloud-aws-2.1.0 --- README.md | 6 +++--- pom.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 06bd352b..acc7dc6f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.0.0.RC1`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.0`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.1). @@ -13,11 +13,11 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| -| 2.1.0-SNAPSHOT | 1.1.0 -> 1.1 | XXXX-XX-XX | +| 2.1.0 | 1.1.0 -> 1.1 | 2014-03-26 | Please read documentation relative to the version you are using: -* [2.1.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.1/README.md) +* [2.1.0](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.0/README.md) ## Generic Configuration diff --git a/pom.xml b/pom.xml index 16a4de68..673f0576 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.1.0-SNAPSHOT + 2.1.0 jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 50a15d8c51341fe2abded0de8156e1dae8d2563c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 19:09:32 +0100 Subject: [PATCH 05/86] prepare for next development iteration --- README.md | 2 ++ pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index acc7dc6f..abff6eae 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,12 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| +| 2.1.1-SNAPSHOT | 1.1.0 -> 1.1 | XXXX-XX-XX | | 2.1.0 | 1.1.0 -> 1.1 | 2014-03-26 | Please read documentation relative to the version you are using: +* [2.1.1-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.1/README.md) * [2.1.0](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.0/README.md) ## Generic Configuration diff --git a/pom.xml b/pom.xml index 673f0576..565fee49 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.1.0 + 2.1.1-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From a9a76e210ba511169b650924222c2fd88dddf6fe Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 18 Apr 2014 11:01:50 +0200 Subject: [PATCH 06/86] cloud-aws 2.1.0 doesn't support elasticsearch 1.1.1 Closes #74. --- README.md | 4 ++-- pom.xml | 4 ++-- src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java | 2 +- .../elasticsearch/repositories/s3/S3SnapshotRestoreTest.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index abff6eae..d4d2024c 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| -| 2.1.1-SNAPSHOT | 1.1.0 -> 1.1 | XXXX-XX-XX | -| 2.1.0 | 1.1.0 -> 1.1 | 2014-03-26 | +| 2.1.1-SNAPSHOT | 1.1.1 -> 1.1 | XXXX-XX-XX | +| 2.1.0 | 1.1.0 | 2014-03-26 | Please read documentation relative to the version you are using: diff --git a/pom.xml b/pom.xml index 565fee49..6d143443 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,8 @@ - 1.1.0 - 4.7.0 + 1.1.1 + 4.7.1 onerror 1 true diff --git a/src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java b/src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java index 49c986ae..1502416d 100644 --- a/src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java +++ b/src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java @@ -49,7 +49,7 @@ public class S3Gateway extends BlobStoreGateway { @Inject public S3Gateway(Settings settings, ThreadPool threadPool, ClusterService clusterService, ClusterName clusterName, AwsS3Service s3Service) throws IOException { - super(settings, threadPool, clusterService); + super(settings, threadPool, clusterService, clusterName); String bucket = componentSettings.get("bucket"); if (bucket == null) { diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index 1484c88c..d936f866 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -142,7 +142,7 @@ public void testSimpleWorkflow() { // Test restore after index deletion logger.info("--> delete indices"); - wipeIndices("test-idx-1", "test-idx-2"); + cluster().wipeIndices("test-idx-1", "test-idx-2"); logger.info("--> restore one index after deletion"); restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); From e98bde4f23e72c8c7bf6ceb15ab28056a05d35aa Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 18 Apr 2014 11:22:37 +0200 Subject: [PATCH 07/86] prepare release elasticsearch-cloud-aws-2.1.1 --- README.md | 6 +++--- pom.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4d2024c..77408c78 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.0`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.1`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.1). @@ -13,12 +13,12 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| -| 2.1.1-SNAPSHOT | 1.1.1 -> 1.1 | XXXX-XX-XX | +| 2.1.1 | 1.1.1 -> 1.1 | 2014-04-18 | | 2.1.0 | 1.1.0 | 2014-03-26 | Please read documentation relative to the version you are using: -* [2.1.1-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.1/README.md) +* [2.1.1](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.1/README.md) * [2.1.0](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.0/README.md) ## Generic Configuration diff --git a/pom.xml b/pom.xml index 6d143443..4471d87a 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.1.1-SNAPSHOT + 2.1.1 jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From fe86cbcc6a27c1a2d60955cd587a663339978b67 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 18 Apr 2014 11:23:33 +0200 Subject: [PATCH 08/86] prepare for next development iteration --- README.md | 2 ++ pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 77408c78..a3b20358 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,13 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| +| 2.1.2-SNAPSHOT | 1.1.1 -> 1.1 | XXXX-XX-XX | | 2.1.1 | 1.1.1 -> 1.1 | 2014-04-18 | | 2.1.0 | 1.1.0 | 2014-03-26 | Please read documentation relative to the version you are using: +* [2.1.2-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.1/README.md) * [2.1.1](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.1/README.md) * [2.1.0](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.0/README.md) diff --git a/pom.xml b/pom.xml index 4471d87a..47a72f0b 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.1.1 + 2.1.2-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 217ef36a7ececfdf7c9c6bec04aefe0cb42dc634 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 29 Apr 2014 21:16:25 +0200 Subject: [PATCH 09/86] create branch es-1.2 (cherry picked from commit 1d7ca8f) --- README.md | 9 +++------ pom.xml | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a3b20358..0d8ee88b 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,18 @@ for the unicast discovery mechanism and add S3 repositories. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.1`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). +* For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.2). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.1). * For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.0). * For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-0.90). | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| -| 2.1.2-SNAPSHOT | 1.1.1 -> 1.1 | XXXX-XX-XX | -| 2.1.1 | 1.1.1 -> 1.1 | 2014-04-18 | -| 2.1.0 | 1.1.0 | 2014-03-26 | +| 2.2.0-SNAPSHOT | 1.2.0 -> 1.2 | XXXX-XX-XX | Please read documentation relative to the version you are using: -* [2.1.2-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.1/README.md) -* [2.1.1](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.1/README.md) -* [2.1.0](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/v2.1.0/README.md) +* [2.2.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.2/README.md) ## Generic Configuration diff --git a/pom.xml b/pom.xml index 47a72f0b..d3dd57dc 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.1.2-SNAPSHOT + 2.2.0-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 1cf68621ff1f22af21a2532b1ba753a759db7c21 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 18 Apr 2014 11:15:54 +0200 Subject: [PATCH 10/86] Remove S3 gateway In elasticsearch core code, we removed Gateways (see https://github.com/elasticsearch/elasticsearch/issues/5422). We need now to remove S3 gateway from aws plugin. Closes #75. (cherry picked from commit d9326af) --- README.md | 27 ---- .../elasticsearch/gateway/s3/S3Gateway.java | 119 ------------------ .../gateway/s3/S3GatewayModule.java | 34 ----- .../index/gateway/s3/S3IndexGateway.java | 50 -------- .../gateway/s3/S3IndexGatewayModule.java | 34 ----- .../index/gateway/s3/S3IndexShardGateway.java | 48 ------- 6 files changed, 312 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java delete mode 100644 src/main/java/org/elasticsearch/gateway/s3/S3GatewayModule.java delete mode 100644 src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGateway.java delete mode 100644 src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGatewayModule.java delete mode 100644 src/main/java/org/elasticsearch/index/gateway/s3/S3IndexShardGateway.java diff --git a/README.md b/README.md index 0d8ee88b..bd18e53c 100644 --- a/README.md +++ b/README.md @@ -76,33 +76,6 @@ One practical use for tag filtering is when an ec2 cluster contains many nodes t Though not dependent on actually using `ec2` as discovery (but still requires the cloud aws plugin installed), the plugin can automatically add node attributes relating to ec2 (for example, availability zone, that can be used with the awareness allocation feature). In order to enable it, set `cloud.node.auto_attributes` to `true` in the settings. -## S3 Gateway - -*note*: As explained [here](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-gateway-s3.html) S3 Gateway functionality is being deprecated. Please use [local gateway](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-gateway-local.html) instead. - -s3 based gateway allows to do long term reliable async persistency of the cluster state and indices directly to Amazon s3. Note, this is a shared gateway where the indices are periodically persisted to s3 while being served from the data location of each node. - -Here is how it can be configured: - - cloud: - aws: - access_key: AKVAIQBF2RECL7FJWGJQ - secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br - - - gateway: - type: s3 - s3: - bucket: bucket_name - -The following are a list of settings (prefixed with `gateway.s3`) that can further control the s3 gateway: - -* `chunk_size`: Big files are broken down into chunks (to overcome AWS 5g limit and use concurrent snapshotting). Default set to `100m`. - -### concurrent_streams - -The `gateway.s3.concurrent_streams` allow to throttle the number of streams (per node) opened against the shared gateway performing the snapshot operation. It defaults to `5`. - ## S3 Repository The S3 repository is using S3 to store snapshots. The S3 repository can be created using the following command: diff --git a/src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java b/src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java deleted file mode 100644 index 1502416d..00000000 --- a/src/main/java/org/elasticsearch/gateway/s3/S3Gateway.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gateway.s3; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchIllegalArgumentException; -import org.elasticsearch.cloud.aws.AwsS3Service; -import org.elasticsearch.cloud.aws.blobstore.S3BlobStore; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.gateway.blobstore.BlobStoreGateway; -import org.elasticsearch.index.gateway.s3.S3IndexGatewayModule; -import org.elasticsearch.threadpool.ThreadPool; - -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * - */ -public class S3Gateway extends BlobStoreGateway { - - private final ExecutorService concurrentStreamPool; - - @Inject - public S3Gateway(Settings settings, ThreadPool threadPool, ClusterService clusterService, - ClusterName clusterName, AwsS3Service s3Service) throws IOException { - super(settings, threadPool, clusterService, clusterName); - - String bucket = componentSettings.get("bucket"); - if (bucket == null) { - throw new ElasticsearchIllegalArgumentException("No bucket defined for s3 gateway"); - } - - String region = componentSettings.get("region"); - if (region == null) { - if (settings.get("cloud.aws.region") != null) { - String regionSetting = settings.get("cloud.aws.region"); - if ("us-east".equals(regionSetting.toLowerCase())) { - region = null; - } else if ("us-east-1".equals(regionSetting.toLowerCase())) { - region = null; - } else if ("us-west".equals(regionSetting.toLowerCase())) { - region = "us-west-1"; - } else if ("us-west-1".equals(regionSetting.toLowerCase())) { - region = "us-west-1"; - } else if ("us-west-2".equals(regionSetting.toLowerCase())) { - region = "us-west-2"; - } else if ("ap-southeast".equals(regionSetting.toLowerCase())) { - region = "ap-southeast-1"; - } else if ("ap-southeast-1".equals(regionSetting.toLowerCase())) { - region = "ap-southeast-1"; - } else if ("ap-southeast-2".equals(regionSetting.toLowerCase())) { - region = "ap-southeast-2"; - } else if ("ap-northeast".equals(regionSetting.toLowerCase())) { - region = "ap-northeast-1"; - } else if ("ap-northeast-1".equals(regionSetting.toLowerCase())) { - region = "ap-northeast-1"; - } else if ("eu-west".equals(regionSetting.toLowerCase())) { - region = "EU"; - } else if ("eu-west-1".equals(regionSetting.toLowerCase())) { - region = "EU"; - } else if ("sa-east".equals(regionSetting.toLowerCase())) { - region = "sa-east-1"; - } else if ("sa-east-1".equals(regionSetting.toLowerCase())) { - region = "sa-east-1"; - } - } - } - ByteSizeValue chunkSize = componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB)); - - int concurrentStreams = componentSettings.getAsInt("concurrent_streams", 5); - this.concurrentStreamPool = EsExecutors.newScaling(1, concurrentStreams, 5, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(settings, "[s3_stream]")); - - logger.debug("using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}]", bucket, region, chunkSize, concurrentStreams); - - initialize(new S3BlobStore(settings, s3Service.client(), bucket, region, concurrentStreamPool), clusterName, chunkSize); - } - - @Override - protected void doClose() throws ElasticsearchException { - super.doClose(); - concurrentStreamPool.shutdown(); - } - - @Override - public String type() { - return "s3"; - } - - @Override - public Class suggestIndexGateway() { - return S3IndexGatewayModule.class; - } -} diff --git a/src/main/java/org/elasticsearch/gateway/s3/S3GatewayModule.java b/src/main/java/org/elasticsearch/gateway/s3/S3GatewayModule.java deleted file mode 100644 index 4a069fec..00000000 --- a/src/main/java/org/elasticsearch/gateway/s3/S3GatewayModule.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gateway.s3; - -import org.elasticsearch.gateway.Gateway; -import org.elasticsearch.gateway.blobstore.BlobStoreGatewayModule; - -/** - * - */ -public class S3GatewayModule extends BlobStoreGatewayModule { - - @Override - protected void configure() { - bind(Gateway.class).to(S3Gateway.class).asEagerSingleton(); - } -} diff --git a/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGateway.java b/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGateway.java deleted file mode 100644 index 87da22f2..00000000 --- a/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGateway.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.gateway.s3; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.gateway.Gateway; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.gateway.IndexShardGateway; -import org.elasticsearch.index.gateway.blobstore.BlobStoreIndexGateway; -import org.elasticsearch.index.settings.IndexSettings; - -/** - * - */ -public class S3IndexGateway extends BlobStoreIndexGateway { - - @Inject - public S3IndexGateway(Index index, @IndexSettings Settings indexSettings, Gateway gateway) { - super(index, indexSettings, gateway); - } - - @Override - public String type() { - return "s3"; - } - - @Override - public Class shardGatewayClass() { - return S3IndexShardGateway.class; - } -} - diff --git a/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGatewayModule.java b/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGatewayModule.java deleted file mode 100644 index 16afc6a1..00000000 --- a/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexGatewayModule.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.gateway.s3; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.gateway.IndexGateway; - -/** - * - */ -public class S3IndexGatewayModule extends AbstractModule { - - @Override - protected void configure() { - bind(IndexGateway.class).to(S3IndexGateway.class).asEagerSingleton(); - } -} diff --git a/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexShardGateway.java b/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexShardGateway.java deleted file mode 100644 index d22dd43c..00000000 --- a/src/main/java/org/elasticsearch/index/gateway/s3/S3IndexShardGateway.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.gateway.s3; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.gateway.IndexGateway; -import org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.service.IndexShard; -import org.elasticsearch.index.store.Store; -import org.elasticsearch.threadpool.ThreadPool; - -/** - * - */ -public class S3IndexShardGateway extends BlobStoreIndexShardGateway { - - @Inject - public S3IndexShardGateway(ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool, IndexGateway indexGateway, - IndexShard indexShard, Store store) { - super(shardId, indexSettings, threadPool, indexGateway, indexShard, store); - } - - @Override - public String type() { - return "s3"; - } -} - From 5be372e3fbf91c386d8d59e690226d153b31d25f Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 18 Apr 2014 11:14:46 +0200 Subject: [PATCH 11/86] Update to elasticsearch 1.2.0 Closes #79. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d3dd57dc..065502df 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,8 @@ - 1.1.1 - 4.7.1 + 1.2.0 + 4.8.0 onerror 1 true From b01f8bfd24102c59e52b650fdb7cbabefb42cc56 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 29 Apr 2014 22:10:32 +0200 Subject: [PATCH 12/86] Update to elasticsearch 1.2.0 / Lucene 4.8 Breaking changes in tests --- pom.xml | 2 +- .../java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java | 2 +- .../elasticsearch/repositories/s3/S3SnapshotRestoreTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 065502df..c4fea3de 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ - 1.2.0 + 1.2.0-SNAPSHOT 4.8.0 onerror 1 diff --git a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java index 5f211422..51232bd4 100644 --- a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java +++ b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java @@ -35,7 +35,7 @@ * This test requires AWS to run. */ @AwsTest -@ClusterScope(scope = Scope.TEST, numNodes = 0) +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class Ec2DiscoveryITest extends AbstractAwsTest { @Test diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index d936f866..040f1f11 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -52,7 +52,7 @@ /** */ @AwsTest -@ClusterScope(scope = Scope.SUITE, numNodes = 2) +@ClusterScope(scope = Scope.SUITE, numDataNodes = 2) public class S3SnapshotRestoreTest extends AbstractAwsTest { @Override From bd006743fb11d2838ae4ea98d1ced19f5c1dc382 Mon Sep 17 00:00:00 2001 From: Kurt Hurtado Date: Tue, 6 May 2014 13:56:42 -0700 Subject: [PATCH 13/86] Adding example S3 bucket permissions, with js syntax --- README.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bd18e53c..8060fe0c 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ ec2 discovery allows to use the ec2 APIs to perform automatic discovery (similar aws: access_key: AKVAIQBF2RECL7FJWGJQ secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br - + discovery: type: ec2 @@ -109,6 +109,77 @@ The S3 repositories are using the same credentials as the rest of the S3 service Multiple S3 repositories can be created. If the buckets require different credentials, then define them as part of the repository settings. +## Recommended S3 Permissions + +In order to restrict the Elasticsearch snapshot process to the minimum required resources, we recommend using Amazon IAM in conjunction with pre-existing S3 buckets. Here is an example policy which will allow the snapshot access to an S3 bucket named "snaps.example.com". This may be configured through the AWS IAM console, by creating a Custom Policy, and using a Policy Document similar to this (changing snaps.example.com to your bucket name). + +```js +{ + "Statement": [ + { + "Action": [ + "s3:ListBucket" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::snaps.example.com" + ] + }, + { + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::snaps.example.com/*" + ] + } + ], + "Version": "2012-10-17" +} + +``` + +You may further restrict the permissions by specifying a prefix within the bucket, in this example, named "foo". + +```js +{ + "Statement": [ + { + "Action": [ + "s3:ListBucket" + ], + "Condition": { + "StringLike": { + "s3:prefix": [ + "foo/*" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::snaps.example.com" + ] + }, + { + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::snaps.example.com/foo/*" + ] + } + ], + "Version": "2012-10-17" +} + +``` + ## Testing Integrations tests in this plugin require working AWS configuration and therefore disabled by default. Three buckets and two iam users have to be created. The first iam user needs access to two buckets in different regions and the final bucket is exclusive for the other iam user. To enable tests prepare a config file elasticsearch.yml with the following content: From 7f4bcce70091ac585cac89e9bec0e08ad2589d56 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Tue, 13 May 2014 17:05:01 +0200 Subject: [PATCH 14/86] Add comment about snapshot/restore if bucket does not exist closes #84 --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 8060fe0c..727dd34f 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,20 @@ You may further restrict the permissions by specifying a prefix within the bucke ``` +The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository registration will fail. If you want elasticsearch to create the bucket instead, you can add the permission to create a specific bucket like this: + +```js +{ + "Action": [ + "s3:CreateBucket" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::snaps.example.com" + ] +} +``` + ## Testing Integrations tests in this plugin require working AWS configuration and therefore disabled by default. Three buckets and two iam users have to be created. The first iam user needs access to two buckets in different regions and the final bucket is exclusive for the other iam user. To enable tests prepare a config file elasticsearch.yml with the following content: From 67ac1490f725fe3fb49ebb075ff682eaac5bce02 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 27 May 2014 16:42:47 +0200 Subject: [PATCH 15/86] S3 Repo documentation Closes #83. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 727dd34f..678a3b0f 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ The following settings are supported: * `chunk_size`: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, i.e. `1g`, `10m`, `5k`. Defaults to `100m`. * `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`. -The S3 repositories are using the same credentials as the rest of the S3 services provided by this plugin (`discovery` and `gateway`). They can be configured the following way: +The S3 repositories are using the same credentials as the rest of the AWSS3 Repo documentation #83 services provided by this plugin (`discovery` and `gateway`). They can be configured the following way: cloud: aws: From 0c39cb9b5bc5b8737909bb5a6c3063ad20053c51 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 28 May 2014 21:28:17 +0200 Subject: [PATCH 16/86] Switch to shared thread pool for snapshot operations Closes #87 --- README.md | 3 +-- pom.xml | 4 ++-- .../cloud/aws/blobstore/S3BlobStore.java | 9 +++++---- .../repositories/s3/S3Repository.java | 10 ++++------ .../repositories/s3/S3SnapshotRestoreTest.java | 14 ++++++++------ 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 678a3b0f..552f8190 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ The S3 repository is using S3 to store snapshots. The S3 repository can be creat $ curl -XPUT 'http://localhost:9200/_snapshot/my_s3_repository' -d '{ "type": "s3", "settings": { - "bucket": "my_backet_name", + "bucket": "my_bucket_name", "region": "us-west" } }' @@ -95,7 +95,6 @@ The following settings are supported: * `base_path`: Specifies the path within bucket to repository data. Defaults to root directory. * `access_key`: The access key to use for authentication. Defaults to value of `cloud.aws.access_key`. * `secret_key`: The secret key to use for authentication. Defaults to value of `cloud.aws.secret_key`. -* `concurrent_streams`: Throttles the number of streams (per node) preforming snapshot operation. Defaults to `5`. * `chunk_size`: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, i.e. `1g`, `10m`, `5k`. Defaults to `100m`. * `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`. diff --git a/pom.xml b/pom.xml index c4fea3de..c2415fcb 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,8 @@ - 1.2.0-SNAPSHOT - 4.8.0 + 1.2.0 + 4.8.1 onerror 1 true diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index 52b4d723..b05ada35 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.concurrent.Executor; @@ -47,16 +48,16 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { private final String region; - private final Executor executor; + private final ThreadPool threadPool; private final int bufferSizeInBytes; - public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, Executor executor) { + public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, ThreadPool threadPool) { super(settings); this.client = client; this.bucket = bucket; this.region = region; - this.executor = executor; + this.threadPool = threadPool; this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); @@ -83,7 +84,7 @@ public String bucket() { } public Executor executor() { - return executor; + return threadPool.executor(ThreadPool.Names.SNAPSHOT_DATA); } public int bufferSizeInBytes() { diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index b9595a3a..cecd8ab3 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -27,12 +27,12 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.snapshots.IndexShardRepository; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositorySettings; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; import java.util.Locale; @@ -74,7 +74,7 @@ public class S3Repository extends BlobStoreRepository { * @throws IOException */ @Inject - public S3Repository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, AwsS3Service s3Service) throws IOException { + public S3Repository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, AwsS3Service s3Service, ThreadPool threadPool) throws IOException { super(name.getName(), repositorySettings, indexShardRepository); String bucket = repositorySettings.settings().get("bucket", componentSettings.get("bucket")); @@ -120,11 +120,9 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, } } } - int concurrentStreams = repositorySettings.settings().getAsInt("concurrent_streams", componentSettings.getAsInt("concurrent_streams", 5)); - ExecutorService concurrentStreamPool = EsExecutors.newScaling(1, concurrentStreams, 5, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(settings, "[s3_stream]")); - logger.debug("using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}]", bucket, region, chunkSize, concurrentStreams); - blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, concurrentStreamPool); + logger.debug("using bucket [{}], region [{}], chunk_size [{}]", bucket, region, chunkSize); + blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, threadPool); this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); String basePath = repositorySettings.settings().get("base_path", null); diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index 040f1f11..4bc16a76 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -180,6 +180,7 @@ public void testRepositoryWithCustomCredentials() { PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() .put("base_path", basePath) + .put("region", bucketSettings.get("region")) .put("access_key", bucketSettings.get("access_key")) .put("secret_key", bucketSettings.get("secret_key")) .put("bucket", bucketSettings.get("bucket")) @@ -292,10 +293,11 @@ public void cleanRepositoryFiles(String basePath) { settings.getByPrefix("repositories.s3.remote-bucket.") }; for (Settings bucket : buckets) { - AmazonS3 client = cluster().getInstance(AwsS3Service.class).client( - bucket.get("region", settings.get("repositories.s3.region")), - bucket.get("access_key", settings.get("cloud.aws.access_key")), - bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); + String region = bucket.get("region", settings.get("repositories.s3.region")); + String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key")); + String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key")); + String bucketName = bucket.get("bucket"); + AmazonS3 client = cluster().getInstance(AwsS3Service.class).client(region, accessKey, secretKey); try { ObjectListing prevListing = null; //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html @@ -308,7 +310,7 @@ public void cleanRepositoryFiles(String basePath) { if (prevListing != null) { list = client.listNextBatchOfObjects(prevListing); } else { - list = client.listObjects(bucket.get("bucket"), basePath); + list = client.listObjects(bucketName, basePath); multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); } for (S3ObjectSummary summary : list.getObjectSummaries()) { @@ -332,7 +334,7 @@ public void cleanRepositoryFiles(String basePath) { client.deleteObjects(multiObjectDeleteRequest); } } catch (Throwable ex) { - logger.warn("Failed to delete S3 repository", ex); + logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region); } } } From 99e5f71e4386acae837df1fb9613efc82a4c201a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 28 May 2014 21:28:17 +0200 Subject: [PATCH 17/86] Update to elasticsearch 1.3.0 Closes #89. --- README.md | 5 +++-- pom.xml | 4 ++-- .../org/elasticsearch/cloud/aws/AbstractAwsTest.java | 10 ++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 552f8190..557b6daf 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ for the unicast discovery mechanism and add S3 repositories. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.1`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). +* For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.3). * For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.2). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.1). * For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.0). @@ -14,11 +15,11 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| -| 2.2.0-SNAPSHOT | 1.2.0 -> 1.2 | XXXX-XX-XX | +| 2.3.0-SNAPSHOT | 1.3.0 -> 1.3 | XXXX-XX-XX | Please read documentation relative to the version you are using: -* [2.2.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.2/README.md) +* [2.3.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.3/README.md) ## Generic Configuration diff --git a/pom.xml b/pom.xml index c2415fcb..10f33a49 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.2.0-SNAPSHOT + 2.3.0-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. @@ -32,7 +32,7 @@ - 1.2.0 + 1.3.0-SNAPSHOT 4.8.1 onerror 1 diff --git a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java index c5f75558..f684cd79 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java @@ -20,6 +20,9 @@ package org.elasticsearch.cloud.aws; import com.carrotsearch.randomizedtesting.annotations.TestGroup; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import java.lang.annotation.Documented; @@ -62,4 +65,11 @@ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest { */ public static final String SYSPROP_AWS = "tests.aws"; + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return ImmutableSettings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .build(); + } } From 1d16ce3d7ad6a343ff0b745e0a6535370b4cf143 Mon Sep 17 00:00:00 2001 From: Wilkes Joiner Date: Wed, 28 May 2014 22:46:30 +0200 Subject: [PATCH 18/86] Add Server Side Encryption support for S3 repository For legal reasons, we need to store our ES backups in S3 using server side encryption. Closes #78. (cherry picked from commit b3f9e12) --- README.md | 3 +- .../cloud/aws/blobstore/S3BlobStore.java | 7 +- .../blobstore/S3ImmutableBlobContainer.java | 3 + .../repositories/s3/S3Repository.java | 7 +- .../s3/S3SnapshotRestoreTest.java | 88 +++++++++++++++++++ 5 files changed, 102 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 557b6daf..c7781619 100644 --- a/README.md +++ b/README.md @@ -98,8 +98,9 @@ The following settings are supported: * `secret_key`: The secret key to use for authentication. Defaults to value of `cloud.aws.secret_key`. * `chunk_size`: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, i.e. `1g`, `10m`, `5k`. Defaults to `100m`. * `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`. +* `server_side_encryption`: When set to `true` files are encrypted on server side using AES256 algorithm. Defaults to `false`. -The S3 repositories are using the same credentials as the rest of the AWSS3 Repo documentation #83 services provided by this plugin (`discovery` and `gateway`). They can be configured the following way: +The S3 repositories are using the same credentials as the rest of the AWS services provided by this plugin (`discovery` and `gateway`). They can be configured the following way: cloud: aws: diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index b05ada35..6db8d53a 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -52,12 +52,15 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { private final int bufferSizeInBytes; - public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, ThreadPool threadPool) { + private final boolean serverSideEncryption; + + public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, ThreadPool threadPool, boolean serverSideEncryption) { super(settings); this.client = client; this.bucket = bucket; this.region = region; this.threadPool = threadPool; + this.serverSideEncryption = serverSideEncryption; this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); @@ -87,6 +90,8 @@ public Executor executor() { return threadPool.executor(ThreadPool.Names.SNAPSHOT_DATA); } + public boolean serverSideEncryption() { return serverSideEncryption; } + public int bufferSizeInBytes() { return bufferSizeInBytes; } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java index b6e6e860..e6bbe8f4 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java @@ -44,6 +44,9 @@ public void writeBlob(final String blobName, final InputStream is, final long si public void run() { try { ObjectMetadata md = new ObjectMetadata(); + if (blobStore.serverSideEncryption()) { + md.setServerSideEncryption(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + } md.setContentLength(sizeInBytes); PutObjectResult objectResult = blobStore.client().putObject(blobStore.bucket(), buildKey(blobName), is, md); listener.onCompleted(); diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index cecd8ab3..48096d32 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -36,8 +36,6 @@ import java.io.IOException; import java.util.Locale; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; /** * Shared file system implementation of the BlobStoreRepository @@ -121,8 +119,9 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, } } - logger.debug("using bucket [{}], region [{}], chunk_size [{}]", bucket, region, chunkSize); - blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, threadPool); + boolean serverSideEncryption = repositorySettings.settings().getAsBoolean("server_side_encryption", componentSettings.getAsBoolean("server_side_encryption", false)); + logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}]", bucket, region, chunkSize, serverSideEncryption); + blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, threadPool, serverSideEncryption); this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); String basePath = repositorySettings.settings().get("base_path", null); diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index 4bc16a76..49790b63 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -44,6 +44,7 @@ import org.junit.Before; import org.junit.Test; +import java.util.List; import java.util.ArrayList; import static org.hamcrest.Matchers.equalTo; @@ -152,6 +153,93 @@ public void testSimpleWorkflow() { assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); } + + @Test + public void testEncryption() { + Client client = client(); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", cluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + .put("chunk_size", randomIntBetween(1000, 10000)) + .put("server_side_encryption", true) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + createIndex("test-idx-1", "test-idx-2", "test-idx-3"); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); + index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); + index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); + } + refresh(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); + assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + + Settings settings = cluster().getInstance(Settings.class); + Settings bucket = settings.getByPrefix("repositories.s3."); + AmazonS3 s3Client = cluster().getInstance(AwsS3Service.class).client( + bucket.get("region", settings.get("repositories.s3.region")), + bucket.get("access_key", settings.get("cloud.aws.access_key")), + bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); + + String bucketName = bucket.get("bucket"); + logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); + List summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries(); + for (S3ObjectSummary summary : summaries) { + assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getServerSideEncryption(), equalTo("AES256")); + } + + logger.info("--> delete some data"); + for (int i = 0; i < 50; i++) { + client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); + } + for (int i = 50; i < 100; i++) { + client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get(); + } + for (int i = 0; i < 100; i += 2) { + client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); + } + refresh(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); + assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); + assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + + logger.info("--> close indices"); + client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); + + logger.info("--> restore all indices from the snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + ensureGreen(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); + assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + + // Test restore after index deletion + logger.info("--> delete indices"); + cluster().wipeIndices("test-idx-1", "test-idx-2"); + logger.info("--> restore one index after deletion"); + restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + ensureGreen(); + assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); + assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); + assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); + } /** * This test verifies that the test configuration is set up in a manner that From 36065723a931e26db5ada15bdb753ac4d92802c7 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 28 May 2014 23:03:54 +0200 Subject: [PATCH 19/86] Cleaning documentation (cherry picked from commit 67c5e39) --- README.md | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c7781619..d4fb844b 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,12 @@ Please read documentation relative to the version you are using: The plugin will automatically use the instance level security credentials (as of 1.7.0), but they can be provided explicitly using `cloud.aws.access_key` and `cloud.aws.secret_key`: - cloud: - aws: - access_key: AKVAIQBF2RECL7FJWGJQ - secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br - +``` +cloud: + aws: + access_key: AKVAIQBF2RECL7FJWGJQ + secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br +``` ### Region @@ -51,13 +52,13 @@ The `cloud.aws.region` can be set to a region and will automatically use the rel ec2 discovery allows to use the ec2 APIs to perform automatic discovery (similar to multicast in non hostile multicast environments). Here is a simple sample configuration: - cloud: - aws: - access_key: AKVAIQBF2RECL7FJWGJQ - secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br +``` +discovery: + type: ec2 +``` - discovery: - type: ec2 +The ec2 discovery is using the same credentials as the rest of the AWS services provided by this plugin (`repositories`). +See [Generic Configuration](#generic-configuration) for details. The following are a list of settings (prefixed with `discovery.ec2`) that can further control the discovery: @@ -81,13 +82,15 @@ Though not dependent on actually using `ec2` as discovery (but still requires th The S3 repository is using S3 to store snapshots. The S3 repository can be created using the following command: - $ curl -XPUT 'http://localhost:9200/_snapshot/my_s3_repository' -d '{ - "type": "s3", - "settings": { - "bucket": "my_bucket_name", - "region": "us-west" - } - }' +```sh +$ curl -XPUT 'http://localhost:9200/_snapshot/my_s3_repository' -d '{ + "type": "s3", + "settings": { + "bucket": "my_bucket_name", + "region": "us-west" + } +}' +``` The following settings are supported: @@ -100,13 +103,8 @@ The following settings are supported: * `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`. * `server_side_encryption`: When set to `true` files are encrypted on server side using AES256 algorithm. Defaults to `false`. -The S3 repositories are using the same credentials as the rest of the AWS services provided by this plugin (`discovery` and `gateway`). They can be configured the following way: - - cloud: - aws: - access_key: AKVAIQBF2RECL7FJWGJQ - secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br - +The S3 repositories are using the same credentials as the rest of the AWS services provided by this plugin (`discovery`). +See [Generic Configuration](#generic-configuration) for details. Multiple S3 repositories can be created. If the buckets require different credentials, then define them as part of the repository settings. From 11e1d8af349506f5534d5e99187db6b28af87e04 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 19 Jun 2014 19:23:15 +0200 Subject: [PATCH 20/86] Update to elasticsearch 1.3.0 Related to #89. --- .../discovery/ec2/Ec2DiscoveryITest.java | 6 +++-- .../s3/S3SnapshotRestoreTest.java | 24 ++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java index 51232bd4..13e85280 100644 --- a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java +++ b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java @@ -23,6 +23,7 @@ import org.elasticsearch.cloud.aws.AbstractAwsTest; import org.elasticsearch.cloud.aws.AbstractAwsTest.AwsTest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import org.junit.Test; @@ -35,16 +36,17 @@ * This test requires AWS to run. */ @AwsTest -@ClusterScope(scope = Scope.TEST, numDataNodes = 0) +@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class Ec2DiscoveryITest extends AbstractAwsTest { @Test public void testStart() { Settings nodeSettings = settingsBuilder() + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .put("cloud.enabled", true) .put("discovery.type", "ec2") .build(); - cluster().startNode(nodeSettings); + internalCluster().startNode(nodeSettings); } } diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index 49790b63..3988c3dc 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; @@ -53,7 +54,7 @@ /** */ @AwsTest -@ClusterScope(scope = Scope.SUITE, numDataNodes = 2) +@ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) public class S3SnapshotRestoreTest extends AbstractAwsTest { @Override @@ -64,6 +65,7 @@ public Settings indexSettings() { .put(MockDirectoryHelper.RANDOM_PREVENT_DOUBLE_WRITE, false) .put(MockDirectoryHelper.RANDOM_NO_DELETE_OPEN_FILE, false) .put("cloud.enabled", true) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .build(); } @@ -85,7 +87,7 @@ public final void wipeAfter() { @Test public void testSimpleWorkflow() { Client client = client(); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", cluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() .put("base_path", basePath) @@ -157,7 +159,7 @@ public void testSimpleWorkflow() { @Test public void testEncryption() { Client client = client(); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", cluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() .put("base_path", basePath) @@ -187,9 +189,9 @@ public void testEncryption() { assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - Settings settings = cluster().getInstance(Settings.class); + Settings settings = internalCluster().getInstance(Settings.class); Settings bucket = settings.getByPrefix("repositories.s3."); - AmazonS3 s3Client = cluster().getInstance(AwsS3Service.class).client( + AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( bucket.get("region", settings.get("repositories.s3.region")), bucket.get("access_key", settings.get("cloud.aws.access_key")), bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); @@ -248,7 +250,7 @@ public void testEncryption() { @Test(expected = UncategorizedExecutionException.class) public void assertRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredentials() { Client client = client(); - Settings bucketSettings = cluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); + Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() @@ -263,7 +265,7 @@ public void assertRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredent @Test public void testRepositoryWithCustomCredentials() { Client client = client(); - Settings bucketSettings = cluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); + Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() @@ -285,7 +287,7 @@ public void testRepositoryWithCustomCredentials() { @Test(expected = UncategorizedExecutionException.class) public void assertRepositoryInRemoteRegionIsRemote() { Client client = client(); - Settings bucketSettings = cluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket."); + Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() @@ -302,7 +304,7 @@ public void assertRepositoryInRemoteRegionIsRemote() { @Test public void testRepositoryInRemoteRegion() { Client client = client(); - Settings settings = cluster().getInstance(Settings.class); + Settings settings = internalCluster().getInstance(Settings.class); Settings bucketSettings = settings.getByPrefix("repositories.s3.remote-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") @@ -374,7 +376,7 @@ public static void wipeRepositories(String... repositories) { * Deletes content of the repository files in the bucket */ public void cleanRepositoryFiles(String basePath) { - Settings settings = cluster().getInstance(Settings.class); + Settings settings = internalCluster().getInstance(Settings.class); Settings[] buckets = { settings.getByPrefix("repositories.s3."), settings.getByPrefix("repositories.s3.private-bucket."), @@ -385,7 +387,7 @@ public void cleanRepositoryFiles(String basePath) { String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key")); String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key")); String bucketName = bucket.get("bucket"); - AmazonS3 client = cluster().getInstance(AwsS3Service.class).client(region, accessKey, secretKey); + AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(region, accessKey, secretKey); try { ObjectListing prevListing = null; //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html From 2f28eda97cddf77bcc637ec700113abb0f147c3b Mon Sep 17 00:00:00 2001 From: Phil Wills Date: Thu, 19 Jun 2014 19:27:28 +0200 Subject: [PATCH 21/86] Elaborate on how to specify AWS credentials Closes #85. (cherry picked from commit 7af1316) --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4fb844b..0fe772a0 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,10 @@ Please read documentation relative to the version you are using: ## Generic Configuration -The plugin will automatically use the instance level security credentials (as of 1.7.0), but they can be provided explicitly using `cloud.aws.access_key` and `cloud.aws.secret_key`: - +The plugin will default to using [IAM Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) credentials +for authentication. These can be overridden by, in increasing order of precedence, system properties `aws.accessKeyId` and `aws.secretKey`, +environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY`, or the elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key`: + ``` cloud: aws: From 6ebfddf7b475585b978312143ab5dd4d639fcdcc Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Thu, 19 Jun 2014 19:35:59 +0200 Subject: [PATCH 22/86] Update README about SNAPSHOT vs released versions Closes #77. (cherry picked from commit a0fa97d) --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0fe772a0..0a79c33e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.1`. +In order to install the plugin, run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.1`. + +where `2.1.1` would be the version applicable to your elasticsearch release, as follows: * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). * For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.3). @@ -13,6 +15,9 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e * For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.0). * For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-0.90). +SNAPSHOT releases are still in development are _NOT_ available for automatic installation with `bin/plugin`. In such cases +you would be responsible building the plugin and providing the file/url yourself. + | AWS Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| | 2.3.0-SNAPSHOT | 1.3.0 -> 1.3 | XXXX-XX-XX | From 73bd1f7af1a3469964cae8d34b3c9b5ab9da880c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 20 Jun 2014 15:22:24 +0200 Subject: [PATCH 23/86] Update to AWS SDK 1.7.13 Closes #97. (cherry picked from commit 4380b84) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 10f33a49..ef8350d3 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ com.amazonaws aws-java-sdk - 1.7.3 + 1.7.13 compile From 580a419f7d5a39d7bcaa68fbf5aa820821c8e967 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 20 Jun 2014 15:30:51 +0200 Subject: [PATCH 24/86] Allow AWS compatible cloud services Users can use their own endpoints for any ec2/s3 compatible API using: `cloud.aws.ec2.endpoint` or `cloud.aws.s3.endpoint` Closes #91. (cherry picked from commit f0fbea5) --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a79c33e..b531b0be 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,11 @@ One practical use for tag filtering is when an ec2 cluster contains many nodes t Though not dependent on actually using `ec2` as discovery (but still requires the cloud aws plugin installed), the plugin can automatically add node attributes relating to ec2 (for example, availability zone, that can be used with the awareness allocation feature). In order to enable it, set `cloud.node.auto_attributes` to `true` in the settings. +### Using other EC2 endpoint + +If you are using any EC2 api compatible service, you can set the endpoint you want to use by setting `cloud.aws.ec2.endpoint` +to your URL provider. + ## S3 Repository The S3 repository is using S3 to store snapshots. The S3 repository can be created using the following command: @@ -115,7 +120,7 @@ See [Generic Configuration](#generic-configuration) for details. Multiple S3 repositories can be created. If the buckets require different credentials, then define them as part of the repository settings. -## Recommended S3 Permissions +### Recommended S3 Permissions In order to restrict the Elasticsearch snapshot process to the minimum required resources, we recommend using Amazon IAM in conjunction with pre-existing S3 buckets. Here is an example policy which will allow the snapshot access to an S3 bucket named "snaps.example.com". This may be configured through the AWS IAM console, by creating a Custom Policy, and using a Policy Document similar to this (changing snaps.example.com to your bucket name). @@ -200,6 +205,12 @@ The bucket needs to exist to register a repository for snapshots. If you did not } ``` +### Using other S3 endpoint + +If you are using any S3 api compatible service, you can set the endpoint you want to use by setting `cloud.aws.s3.endpoint` +to your URL provider. + + ## Testing Integrations tests in this plugin require working AWS configuration and therefore disabled by default. Three buckets and two iam users have to be created. The first iam user needs access to two buckets in different regions and the final bucket is exclusive for the other iam user. To enable tests prepare a config file elasticsearch.yml with the following content: From 37c7e86a820fbf239e3d8618b3aa205656e2439e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sat, 21 Jun 2014 16:46:30 +0200 Subject: [PATCH 25/86] Added retry mechanism for S3 connection errors Closes #95 (cherry picked from commit 11f4e9c) --- README.md | 1 + .../elasticsearch/cloud/aws/AwsModule.java | 18 +- .../elasticsearch/cloud/aws/AwsS3Service.java | 170 +----- .../cloud/aws/InternalAwsS3Service.java | 195 ++++++ .../blobstore/AbstractS3BlobContainer.java | 22 +- .../cloud/aws/blobstore/S3BlobStore.java | 8 +- .../blobstore/S3ImmutableBlobContainer.java | 41 +- .../plugin/cloud/aws/CloudAwsPlugin.java | 8 +- .../cloud/aws/AbstractAwsTest.java | 3 + .../cloud/aws/AmazonS3Wrapper.java | 554 ++++++++++++++++++ .../elasticsearch/cloud/aws/TestAmazonS3.java | 115 ++++ .../cloud/aws/TestAwsS3Service.java | 68 +++ 12 files changed, 1012 insertions(+), 191 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java create mode 100644 src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java create mode 100644 src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java create mode 100644 src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java diff --git a/README.md b/README.md index b531b0be..64feb0b9 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ The following settings are supported: * `chunk_size`: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, i.e. `1g`, `10m`, `5k`. Defaults to `100m`. * `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`. * `server_side_encryption`: When set to `true` files are encrypted on server side using AES256 algorithm. Defaults to `false`. +* `max_retries`: Number of retries in case of S3 errors. Defaults to `3`. The S3 repositories are using the same credentials as the rest of the AWS services provided by this plugin (`discovery`). See [Generic Configuration](#generic-configuration) for details. diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsModule.java b/src/main/java/org/elasticsearch/cloud/aws/AwsModule.java index d0eaf1d8..254fe6fb 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsModule.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsModule.java @@ -20,15 +20,29 @@ package org.elasticsearch.cloud.aws; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.settings.Settings; /** * */ public class AwsModule extends AbstractModule { + private final Settings settings; + + public static final String S3_SERVICE_TYPE_KEY = "cloud.aws.s3service.type"; + + public AwsModule(Settings settings) { + this.settings = settings; + } + @Override protected void configure() { - bind(AwsS3Service.class).asEagerSingleton(); + bind(AwsS3Service.class).to(getS3ServiceClass(settings)).asEagerSingleton(); bind(AwsEc2Service.class).asEagerSingleton(); } -} + + public static Class getS3ServiceClass(Settings settings) { + return settings.getAsClass(S3_SERVICE_TYPE_KEY, InternalAwsS3Service.class); + } + +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index 80127db4..af014767 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -19,176 +19,14 @@ package org.elasticsearch.cloud.aws; -import java.util.HashMap; -import java.util.Map; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.Protocol; -import com.amazonaws.auth.*; -import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchIllegalArgumentException; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.component.LifecycleComponent; /** * */ -public class AwsS3Service extends AbstractLifecycleComponent { - - /** - * (acceskey, endpoint) -> client - */ - private Map, AmazonS3Client> clients = new HashMap, AmazonS3Client>(); - - @Inject - public AwsS3Service(Settings settings, SettingsFilter settingsFilter) { - super(settings); - - settingsFilter.addFilter(new AwsSettingsFilter()); - } - - public synchronized AmazonS3 client() { - String endpoint = getDefaultEndpoint(); - String account = componentSettings.get("access_key", settings.get("cloud.account")); - String key = componentSettings.get("secret_key", settings.get("cloud.key")); - - return getClient(endpoint, account, key); - } - - public synchronized AmazonS3 client(String region, String account, String key) { - String endpoint; - if (region == null) { - endpoint = getDefaultEndpoint(); - } else { - endpoint = getEndpoint(region); - logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); - } - if (account == null || key == null) { - account = componentSettings.get("access_key", settings.get("cloud.account")); - key = componentSettings.get("secret_key", settings.get("cloud.key")); - } - - return getClient(endpoint, account, key); - } - - - private synchronized AmazonS3 getClient(String endpoint, String account, String key) { - Tuple clientDescriptor = new Tuple(endpoint, account); - AmazonS3Client client = clients.get(clientDescriptor); - if (client != null) { - return client; - } - - ClientConfiguration clientConfiguration = new ClientConfiguration(); - String protocol = componentSettings.get("protocol", "http").toLowerCase(); - if ("http".equals(protocol)) { - clientConfiguration.setProtocol(Protocol.HTTP); - } else if ("https".equals(protocol)) { - clientConfiguration.setProtocol(Protocol.HTTPS); - } else { - throw new ElasticsearchIllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]"); - } - - String proxyHost = componentSettings.get("proxy_host"); - if (proxyHost != null) { - String portString = componentSettings.get("proxy_port", "80"); - Integer proxyPort; - try { - proxyPort = Integer.parseInt(portString, 10); - } catch (NumberFormatException ex) { - throw new ElasticsearchIllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex); - } - clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); - } - - AWSCredentialsProvider credentials; - - if (account == null && key == null) { - credentials = new AWSCredentialsProviderChain( - new EnvironmentVariableCredentialsProvider(), - new SystemPropertiesCredentialsProvider(), - new InstanceProfileCredentialsProvider() - ); - } else { - credentials = new AWSCredentialsProviderChain( - new StaticCredentialsProvider(new BasicAWSCredentials(account, key)) - ); - } - client = new AmazonS3Client(credentials, clientConfiguration); - - if (endpoint != null) { - client.setEndpoint(endpoint); - } - clients.put(clientDescriptor, client); - return client; - } - - private String getDefaultEndpoint() { - String endpoint = null; - if (componentSettings.get("s3.endpoint") != null) { - endpoint = componentSettings.get("s3.endpoint"); - logger.debug("using explicit s3 endpoint [{}]", endpoint); - } else if (componentSettings.get("region") != null) { - String region = componentSettings.get("region").toLowerCase(); - endpoint = getEndpoint(region); - logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); - } - return endpoint; - } - - private static String getEndpoint(String region) { - if ("us-east".equals(region)) { - return "s3.amazonaws.com"; - } else if ("us-east-1".equals(region)) { - return "s3.amazonaws.com"; - } else if ("us-west".equals(region)) { - return "s3-us-west-1.amazonaws.com"; - } else if ("us-west-1".equals(region)) { - return "s3-us-west-1.amazonaws.com"; - } else if ("us-west-2".equals(region)) { - return "s3-us-west-2.amazonaws.com"; - } else if ("ap-southeast".equals(region)) { - return "s3-ap-southeast-1.amazonaws.com"; - } else if ("ap-southeast-1".equals(region)) { - return "s3-ap-southeast-1.amazonaws.com"; - } else if ("ap-southeast-2".equals(region)) { - return "s3-ap-southeast-2.amazonaws.com"; - } else if ("ap-northeast".equals(region)) { - return "s3-ap-northeast-1.amazonaws.com"; - } else if ("ap-northeast-1".equals(region)) { - return "s3-ap-northeast-1.amazonaws.com"; - } else if ("eu-west".equals(region)) { - return "s3-eu-west-1.amazonaws.com"; - } else if ("eu-west-1".equals(region)) { - return "s3-eu-west-1.amazonaws.com"; - } else if ("sa-east".equals(region)) { - return "s3-sa-east-1.amazonaws.com"; - } else if ("sa-east-1".equals(region)) { - return "s3-sa-east-1.amazonaws.com"; - } else { - throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); - } - } - - @Override - protected void doStart() throws ElasticsearchException { - } - - @Override - protected void doStop() throws ElasticsearchException { - } - - @Override - protected void doClose() throws ElasticsearchException { - for (AmazonS3Client client : clients.values()) { - client.shutdown(); - } - } +public interface AwsS3Service extends LifecycleComponent { + AmazonS3 client(); + AmazonS3 client(String region, String account, String key); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java new file mode 100644 index 00000000..9cf21580 --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -0,0 +1,195 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws; + +import java.util.HashMap; +import java.util.Map; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.*; +import com.amazonaws.internal.StaticCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; + +/** + * + */ +public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { + + /** + * (acceskey, endpoint) -> client + */ + private Map, AmazonS3Client> clients = new HashMap, AmazonS3Client>(); + + @Inject + public InternalAwsS3Service(Settings settings, SettingsFilter settingsFilter) { + super(settings); + + settingsFilter.addFilter(new AwsSettingsFilter()); + } + + @Override + public synchronized AmazonS3 client() { + String endpoint = getDefaultEndpoint(); + String account = componentSettings.get("access_key", settings.get("cloud.account")); + String key = componentSettings.get("secret_key", settings.get("cloud.key")); + + return getClient(endpoint, account, key); + } + + @Override + public synchronized AmazonS3 client(String region, String account, String key) { + String endpoint; + if (region == null) { + endpoint = getDefaultEndpoint(); + } else { + endpoint = getEndpoint(region); + logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + } + if (account == null || key == null) { + account = componentSettings.get("access_key", settings.get("cloud.account")); + key = componentSettings.get("secret_key", settings.get("cloud.key")); + } + + return getClient(endpoint, account, key); + } + + + private synchronized AmazonS3 getClient(String endpoint, String account, String key) { + Tuple clientDescriptor = new Tuple(endpoint, account); + AmazonS3Client client = clients.get(clientDescriptor); + if (client != null) { + return client; + } + + ClientConfiguration clientConfiguration = new ClientConfiguration(); + String protocol = componentSettings.get("protocol", "http").toLowerCase(); + if ("http".equals(protocol)) { + clientConfiguration.setProtocol(Protocol.HTTP); + } else if ("https".equals(protocol)) { + clientConfiguration.setProtocol(Protocol.HTTPS); + } else { + throw new ElasticsearchIllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]"); + } + + String proxyHost = componentSettings.get("proxy_host"); + if (proxyHost != null) { + String portString = componentSettings.get("proxy_port", "80"); + Integer proxyPort; + try { + proxyPort = Integer.parseInt(portString, 10); + } catch (NumberFormatException ex) { + throw new ElasticsearchIllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex); + } + clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); + } + + AWSCredentialsProvider credentials; + + if (account == null && key == null) { + credentials = new AWSCredentialsProviderChain( + new EnvironmentVariableCredentialsProvider(), + new SystemPropertiesCredentialsProvider(), + new InstanceProfileCredentialsProvider() + ); + } else { + credentials = new AWSCredentialsProviderChain( + new StaticCredentialsProvider(new BasicAWSCredentials(account, key)) + ); + } + client = new AmazonS3Client(credentials, clientConfiguration); + + if (endpoint != null) { + client.setEndpoint(endpoint); + } + clients.put(clientDescriptor, client); + return client; + } + + private String getDefaultEndpoint() { + String endpoint = null; + if (componentSettings.get("s3.endpoint") != null) { + endpoint = componentSettings.get("s3.endpoint"); + logger.debug("using explicit s3 endpoint [{}]", endpoint); + } else if (componentSettings.get("region") != null) { + String region = componentSettings.get("region").toLowerCase(); + endpoint = getEndpoint(region); + logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + } + return endpoint; + } + + private static String getEndpoint(String region) { + if ("us-east".equals(region)) { + return "s3.amazonaws.com"; + } else if ("us-east-1".equals(region)) { + return "s3.amazonaws.com"; + } else if ("us-west".equals(region)) { + return "s3-us-west-1.amazonaws.com"; + } else if ("us-west-1".equals(region)) { + return "s3-us-west-1.amazonaws.com"; + } else if ("us-west-2".equals(region)) { + return "s3-us-west-2.amazonaws.com"; + } else if ("ap-southeast".equals(region)) { + return "s3-ap-southeast-1.amazonaws.com"; + } else if ("ap-southeast-1".equals(region)) { + return "s3-ap-southeast-1.amazonaws.com"; + } else if ("ap-southeast-2".equals(region)) { + return "s3-ap-southeast-2.amazonaws.com"; + } else if ("ap-northeast".equals(region)) { + return "s3-ap-northeast-1.amazonaws.com"; + } else if ("ap-northeast-1".equals(region)) { + return "s3-ap-northeast-1.amazonaws.com"; + } else if ("eu-west".equals(region)) { + return "s3-eu-west-1.amazonaws.com"; + } else if ("eu-west-1".equals(region)) { + return "s3-eu-west-1.amazonaws.com"; + } else if ("sa-east".equals(region)) { + return "s3-sa-east-1.amazonaws.com"; + } else if ("sa-east-1".equals(region)) { + return "s3-sa-east-1.amazonaws.com"; + } else { + throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); + } + } + + @Override + protected void doStart() throws ElasticsearchException { + } + + @Override + protected void doStop() throws ElasticsearchException { + } + + @Override + protected void doClose() throws ElasticsearchException { + for (AmazonS3Client client : clients.values()) { + client.shutdown(); + } + } +} diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java index 5c44a52c..5fd28435 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java @@ -19,12 +19,15 @@ package org.elasticsearch.cloud.aws.blobstore; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStoreException; import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; import org.elasticsearch.common.collect.ImmutableMap; @@ -56,8 +59,10 @@ public boolean blobExists(String blobName) { try { blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName)); return true; - } catch (Exception e) { + } catch (AmazonS3Exception e) { return false; + } catch (Throwable e) { + throw new BlobStoreException("failed to check if blob exists", e); } } @@ -76,7 +81,7 @@ public void run() { try { S3Object object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); is = object.getObjectContent(); - } catch (Exception e) { + } catch (Throwable e) { listener.onFailure(e); return; } @@ -87,12 +92,8 @@ public void run() { listener.onPartial(buffer, 0, bytesRead); } listener.onCompleted(); - } catch (Exception e) { - try { - is.close(); - } catch (IOException e1) { - // ignore - } + } catch (Throwable e) { + IOUtils.closeWhileHandlingException(is); listener.onFailure(e); } } @@ -135,4 +136,9 @@ public ImmutableMap listBlobs() throws IOException { protected String buildKey(String blobName) { return keyPath + blobName; } + + protected boolean shouldRetry(AmazonS3Exception e) { + return e.getStatusCode() == 400 && "RequestTimeout".equals(e.getErrorCode()); + } + } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index 6db8d53a..e060fb55 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -54,6 +54,8 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { private final boolean serverSideEncryption; + private final int numberOfRetries; + public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, ThreadPool threadPool, boolean serverSideEncryption) { super(settings); this.client = client; @@ -63,7 +65,7 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable this.serverSideEncryption = serverSideEncryption; this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); - + this.numberOfRetries = settings.getAsInt("max_retries", 3); if (!client.doesBucketExist(bucket)) { if (region != null) { client.createBucket(bucket, region); @@ -96,6 +98,10 @@ public int bufferSizeInBytes() { return bufferSizeInBytes; } + public int numberOfRetries() { + return numberOfRetries; + } + @Override public ImmutableBlobContainer immutableBlobContainer(BlobPath path) { return new S3ImmutableBlobContainer(path, this); diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java index e6bbe8f4..9ea97bd5 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java @@ -19,8 +19,8 @@ package org.elasticsearch.cloud.aws.blobstore; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectResult; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.ImmutableBlobContainer; import org.elasticsearch.common.blobstore.support.BlobStores; @@ -42,16 +42,37 @@ public void writeBlob(final String blobName, final InputStream is, final long si blobStore.executor().execute(new Runnable() { @Override public void run() { - try { - ObjectMetadata md = new ObjectMetadata(); - if (blobStore.serverSideEncryption()) { - md.setServerSideEncryption(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + int retry = 0; + // Read limit is ignored by InputStreamIndexInput, but we will set it anyway in case + // implementation will change + is.mark(Integer.MAX_VALUE); + while (true) { + try { + ObjectMetadata md = new ObjectMetadata(); + if (blobStore.serverSideEncryption()) { + md.setServerSideEncryption(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + } + md.setContentLength(sizeInBytes); + blobStore.client().putObject(blobStore.bucket(), buildKey(blobName), is, md); + listener.onCompleted(); + return; + } catch (AmazonS3Exception e) { + if (shouldRetry(e) && retry < blobStore.numberOfRetries()) { + try { + is.reset(); + } catch (IOException ex) { + listener.onFailure(e); + return; + } + retry++; + } else { + listener.onFailure(e); + return; + } + } catch (Throwable e) { + listener.onFailure(e); + return; } - md.setContentLength(sizeInBytes); - PutObjectResult objectResult = blobStore.client().putObject(blobStore.bucket(), buildKey(blobName), is, md); - listener.onCompleted(); - } catch (Exception e) { - listener.onFailure(e); } } }); diff --git a/src/main/java/org/elasticsearch/plugin/cloud/aws/CloudAwsPlugin.java b/src/main/java/org/elasticsearch/plugin/cloud/aws/CloudAwsPlugin.java index dd773c4d..8d843854 100644 --- a/src/main/java/org/elasticsearch/plugin/cloud/aws/CloudAwsPlugin.java +++ b/src/main/java/org/elasticsearch/plugin/cloud/aws/CloudAwsPlugin.java @@ -55,10 +55,10 @@ public String description() { } @Override - public Collection> modules() { - Collection> modules = Lists.newArrayList(); + public Collection modules(Settings settings) { + Collection modules = Lists.newArrayList(); if (settings.getAsBoolean("cloud.enabled", true)) { - modules.add(AwsModule.class); + modules.add(new AwsModule(settings)); } return modules; } @@ -67,7 +67,7 @@ public Collection> modules() { public Collection> services() { Collection> services = Lists.newArrayList(); if (settings.getAsBoolean("cloud.enabled", true)) { - services.add(AwsS3Service.class); + services.add(AwsModule.getS3ServiceClass(settings)); services.add(AwsEc2Service.class); } return services; diff --git a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java index f684cd79..98c79461 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java @@ -70,6 +70,9 @@ protected Settings nodeSettings(int nodeOrdinal) { return ImmutableSettings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put( AwsModule.S3_SERVICE_TYPE_KEY, TestAwsS3Service.class) + .put("cloud.aws.test.random", randomInt()) + .put("cloud.aws.test.write_failures", 0.1) .build(); } } diff --git a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java new file mode 100644 index 00000000..950aa72a --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -0,0 +1,554 @@ +/* + * Licensed to Elasticsearch (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.AmazonWebServiceRequest; +import com.amazonaws.HttpMethod; +import com.amazonaws.regions.Region; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.S3ClientOptions; +import com.amazonaws.services.s3.S3ResponseMetadata; +import com.amazonaws.services.s3.model.*; + +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.util.Date; +import java.util.List; + +/** + * + */ +public class AmazonS3Wrapper implements AmazonS3 { + + protected AmazonS3 delegate; + + public AmazonS3Wrapper(AmazonS3 delegate) { + this.delegate = delegate; + } + + + @Override + public void setEndpoint(String endpoint) { + delegate.setEndpoint(endpoint); + } + + @Override + public void setRegion(Region region) throws IllegalArgumentException { + delegate.setRegion(region); + } + + @Override + public void setS3ClientOptions(S3ClientOptions clientOptions) { + delegate.setS3ClientOptions(clientOptions); + } + + @Override + public void changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass) throws AmazonClientException, AmazonServiceException { + delegate.changeObjectStorageClass(bucketName, key, newStorageClass); + } + + @Override + public void setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation) throws AmazonClientException, AmazonServiceException { + delegate.setObjectRedirectLocation(bucketName, key, newRedirectLocation); + } + + @Override + public ObjectListing listObjects(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.listObjects(bucketName); + } + + @Override + public ObjectListing listObjects(String bucketName, String prefix) throws AmazonClientException, AmazonServiceException { + return delegate.listObjects(bucketName, prefix); + } + + @Override + public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException { + return delegate.listObjects(listObjectsRequest); + } + + @Override + public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws AmazonClientException, AmazonServiceException { + return delegate.listNextBatchOfObjects(previousObjectListing); + } + + @Override + public VersionListing listVersions(String bucketName, String prefix) throws AmazonClientException, AmazonServiceException { + return delegate.listVersions(bucketName, prefix); + } + + @Override + public VersionListing listNextBatchOfVersions(VersionListing previousVersionListing) throws AmazonClientException, AmazonServiceException { + return delegate.listNextBatchOfVersions(previousVersionListing); + } + + @Override + public VersionListing listVersions(String bucketName, String prefix, String keyMarker, String versionIdMarker, String delimiter, Integer maxResults) throws AmazonClientException, AmazonServiceException { + return delegate.listVersions(bucketName, prefix, keyMarker, versionIdMarker, delimiter, maxResults); + } + + @Override + public VersionListing listVersions(ListVersionsRequest listVersionsRequest) throws AmazonClientException, AmazonServiceException { + return delegate.listVersions(listVersionsRequest); + } + + @Override + public Owner getS3AccountOwner() throws AmazonClientException, AmazonServiceException { + return delegate.getS3AccountOwner(); + } + + @Override + public boolean doesBucketExist(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.doesBucketExist(bucketName); + } + + @Override + public List listBuckets() throws AmazonClientException, AmazonServiceException { + return delegate.listBuckets(); + } + + @Override + public List listBuckets(ListBucketsRequest listBucketsRequest) throws AmazonClientException, AmazonServiceException { + return delegate.listBuckets(listBucketsRequest); + } + + @Override + public String getBucketLocation(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketLocation(bucketName); + } + + @Override + public String getBucketLocation(GetBucketLocationRequest getBucketLocationRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketLocation(getBucketLocationRequest); + } + + @Override + public Bucket createBucket(CreateBucketRequest createBucketRequest) throws AmazonClientException, AmazonServiceException { + return delegate.createBucket(createBucketRequest); + } + + @Override + public Bucket createBucket(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.createBucket(bucketName); + } + + @Override + public Bucket createBucket(String bucketName, com.amazonaws.services.s3.model.Region region) throws AmazonClientException, AmazonServiceException { + return delegate.createBucket(bucketName, region); + } + + @Override + public Bucket createBucket(String bucketName, String region) throws AmazonClientException, AmazonServiceException { + return delegate.createBucket(bucketName, region); + } + + @Override + public AccessControlList getObjectAcl(String bucketName, String key) throws AmazonClientException, AmazonServiceException { + return delegate.getObjectAcl(bucketName, key); + } + + @Override + public AccessControlList getObjectAcl(String bucketName, String key, String versionId) throws AmazonClientException, AmazonServiceException { + return delegate.getObjectAcl(bucketName, key, versionId); + } + + @Override + public void setObjectAcl(String bucketName, String key, AccessControlList acl) throws AmazonClientException, AmazonServiceException { + delegate.setObjectAcl(bucketName, key, acl); + } + + @Override + public void setObjectAcl(String bucketName, String key, CannedAccessControlList acl) throws AmazonClientException, AmazonServiceException { + delegate.setObjectAcl(bucketName, key, acl); + } + + @Override + public void setObjectAcl(String bucketName, String key, String versionId, AccessControlList acl) throws AmazonClientException, AmazonServiceException { + delegate.setObjectAcl(bucketName, key, versionId, acl); + } + + @Override + public void setObjectAcl(String bucketName, String key, String versionId, CannedAccessControlList acl) throws AmazonClientException, AmazonServiceException { + delegate.setObjectAcl(bucketName, key, versionId, acl); + } + + @Override + public AccessControlList getBucketAcl(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketAcl(bucketName); + } + + @Override + public void setBucketAcl(SetBucketAclRequest setBucketAclRequest) throws AmazonClientException, AmazonServiceException { + delegate.setBucketAcl(setBucketAclRequest); + } + + @Override + public AccessControlList getBucketAcl(GetBucketAclRequest getBucketAclRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketAcl(getBucketAclRequest); + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList acl) throws AmazonClientException, AmazonServiceException { + delegate.setBucketAcl(bucketName, acl); + } + + @Override + public void setBucketAcl(String bucketName, CannedAccessControlList acl) throws AmazonClientException, AmazonServiceException { + delegate.setBucketAcl(bucketName, acl); + } + + @Override + public ObjectMetadata getObjectMetadata(String bucketName, String key) throws AmazonClientException, AmazonServiceException { + return delegate.getObjectMetadata(bucketName, key); + } + + @Override + public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getObjectMetadata(getObjectMetadataRequest); + } + + @Override + public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException { + return delegate.getObject(bucketName, key); + } + + @Override + public S3Object getObject(GetObjectRequest getObjectRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getObject(getObjectRequest); + } + + @Override + public ObjectMetadata getObject(GetObjectRequest getObjectRequest, File destinationFile) throws AmazonClientException, AmazonServiceException { + return delegate.getObject(getObjectRequest, destinationFile); + } + + @Override + public void deleteBucket(DeleteBucketRequest deleteBucketRequest) throws AmazonClientException, AmazonServiceException { + delegate.deleteBucket(deleteBucketRequest); + } + + @Override + public void deleteBucket(String bucketName) throws AmazonClientException, AmazonServiceException { + delegate.deleteBucket(bucketName); + } + + @Override + public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException { + return delegate.putObject(putObjectRequest); + } + + @Override + public PutObjectResult putObject(String bucketName, String key, File file) throws AmazonClientException, AmazonServiceException { + return delegate.putObject(bucketName, key, file); + } + + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) throws AmazonClientException, AmazonServiceException { + return delegate.putObject(bucketName, key, input, metadata); + } + + @Override + public CopyObjectResult copyObject(String sourceBucketName, String sourceKey, String destinationBucketName, String destinationKey) throws AmazonClientException, AmazonServiceException { + return delegate.copyObject(sourceBucketName, sourceKey, destinationBucketName, destinationKey); + } + + @Override + public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) throws AmazonClientException, AmazonServiceException { + return delegate.copyObject(copyObjectRequest); + } + + @Override + public CopyPartResult copyPart(CopyPartRequest copyPartRequest) throws AmazonClientException, AmazonServiceException { + return delegate.copyPart(copyPartRequest); + } + + @Override + public void deleteObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException { + delegate.deleteObject(bucketName, key); + } + + @Override + public void deleteObject(DeleteObjectRequest deleteObjectRequest) throws AmazonClientException, AmazonServiceException { + delegate.deleteObject(deleteObjectRequest); + } + + @Override + public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) throws AmazonClientException, AmazonServiceException { + return delegate.deleteObjects(deleteObjectsRequest); + } + + @Override + public void deleteVersion(String bucketName, String key, String versionId) throws AmazonClientException, AmazonServiceException { + delegate.deleteVersion(bucketName, key, versionId); + } + + @Override + public void deleteVersion(DeleteVersionRequest deleteVersionRequest) throws AmazonClientException, AmazonServiceException { + delegate.deleteVersion(deleteVersionRequest); + } + + @Override + public BucketLoggingConfiguration getBucketLoggingConfiguration(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketLoggingConfiguration(bucketName); + } + + @Override + public void setBucketLoggingConfiguration(SetBucketLoggingConfigurationRequest setBucketLoggingConfigurationRequest) throws AmazonClientException, AmazonServiceException { + delegate.setBucketLoggingConfiguration(setBucketLoggingConfigurationRequest); + } + + @Override + public BucketVersioningConfiguration getBucketVersioningConfiguration(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketVersioningConfiguration(bucketName); + } + + @Override + public void setBucketVersioningConfiguration(SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest) throws AmazonClientException, AmazonServiceException { + delegate.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest); + } + + @Override + public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucketName) { + return delegate.getBucketLifecycleConfiguration(bucketName); + } + + @Override + public void setBucketLifecycleConfiguration(String bucketName, BucketLifecycleConfiguration bucketLifecycleConfiguration) { + delegate.setBucketLifecycleConfiguration(bucketName, bucketLifecycleConfiguration); + } + + @Override + public void setBucketLifecycleConfiguration(SetBucketLifecycleConfigurationRequest setBucketLifecycleConfigurationRequest) { + delegate.setBucketLifecycleConfiguration(setBucketLifecycleConfigurationRequest); + } + + @Override + public void deleteBucketLifecycleConfiguration(String bucketName) { + delegate.deleteBucketLifecycleConfiguration(bucketName); + } + + @Override + public void deleteBucketLifecycleConfiguration(DeleteBucketLifecycleConfigurationRequest deleteBucketLifecycleConfigurationRequest) { + delegate.deleteBucketLifecycleConfiguration(deleteBucketLifecycleConfigurationRequest); + } + + @Override + public BucketCrossOriginConfiguration getBucketCrossOriginConfiguration(String bucketName) { + return delegate.getBucketCrossOriginConfiguration(bucketName); + } + + @Override + public void setBucketCrossOriginConfiguration(String bucketName, BucketCrossOriginConfiguration bucketCrossOriginConfiguration) { + delegate.setBucketCrossOriginConfiguration(bucketName, bucketCrossOriginConfiguration); + } + + @Override + public void setBucketCrossOriginConfiguration(SetBucketCrossOriginConfigurationRequest setBucketCrossOriginConfigurationRequest) { + delegate.setBucketCrossOriginConfiguration(setBucketCrossOriginConfigurationRequest); + } + + @Override + public void deleteBucketCrossOriginConfiguration(String bucketName) { + delegate.deleteBucketCrossOriginConfiguration(bucketName); + } + + @Override + public void deleteBucketCrossOriginConfiguration(DeleteBucketCrossOriginConfigurationRequest deleteBucketCrossOriginConfigurationRequest) { + delegate.deleteBucketCrossOriginConfiguration(deleteBucketCrossOriginConfigurationRequest); + } + + @Override + public BucketTaggingConfiguration getBucketTaggingConfiguration(String bucketName) { + return delegate.getBucketTaggingConfiguration(bucketName); + } + + @Override + public void setBucketTaggingConfiguration(String bucketName, BucketTaggingConfiguration bucketTaggingConfiguration) { + delegate.setBucketTaggingConfiguration(bucketName, bucketTaggingConfiguration); + } + + @Override + public void setBucketTaggingConfiguration(SetBucketTaggingConfigurationRequest setBucketTaggingConfigurationRequest) { + delegate.setBucketTaggingConfiguration(setBucketTaggingConfigurationRequest); + } + + @Override + public void deleteBucketTaggingConfiguration(String bucketName) { + delegate.deleteBucketTaggingConfiguration(bucketName); + } + + @Override + public void deleteBucketTaggingConfiguration(DeleteBucketTaggingConfigurationRequest deleteBucketTaggingConfigurationRequest) { + delegate.deleteBucketTaggingConfiguration(deleteBucketTaggingConfigurationRequest); + } + + @Override + public BucketNotificationConfiguration getBucketNotificationConfiguration(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketNotificationConfiguration(bucketName); + } + + @Override + public void setBucketNotificationConfiguration(SetBucketNotificationConfigurationRequest setBucketNotificationConfigurationRequest) throws AmazonClientException, AmazonServiceException { + delegate.setBucketNotificationConfiguration(setBucketNotificationConfigurationRequest); + } + + @Override + public void setBucketNotificationConfiguration(String bucketName, BucketNotificationConfiguration bucketNotificationConfiguration) throws AmazonClientException, AmazonServiceException { + delegate.setBucketNotificationConfiguration(bucketName, bucketNotificationConfiguration); + } + + @Override + public BucketWebsiteConfiguration getBucketWebsiteConfiguration(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketWebsiteConfiguration(bucketName); + } + + @Override + public BucketWebsiteConfiguration getBucketWebsiteConfiguration(GetBucketWebsiteConfigurationRequest getBucketWebsiteConfigurationRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketWebsiteConfiguration(getBucketWebsiteConfigurationRequest); + } + + @Override + public void setBucketWebsiteConfiguration(String bucketName, BucketWebsiteConfiguration configuration) throws AmazonClientException, AmazonServiceException { + delegate.setBucketWebsiteConfiguration(bucketName, configuration); + } + + @Override + public void setBucketWebsiteConfiguration(SetBucketWebsiteConfigurationRequest setBucketWebsiteConfigurationRequest) throws AmazonClientException, AmazonServiceException { + delegate.setBucketWebsiteConfiguration(setBucketWebsiteConfigurationRequest); + } + + @Override + public void deleteBucketWebsiteConfiguration(String bucketName) throws AmazonClientException, AmazonServiceException { + delegate.deleteBucketWebsiteConfiguration(bucketName); + } + + @Override + public void deleteBucketWebsiteConfiguration(DeleteBucketWebsiteConfigurationRequest deleteBucketWebsiteConfigurationRequest) throws AmazonClientException, AmazonServiceException { + delegate.deleteBucketWebsiteConfiguration(deleteBucketWebsiteConfigurationRequest); + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketPolicy(bucketName); + } + + @Override + public BucketPolicy getBucketPolicy(GetBucketPolicyRequest getBucketPolicyRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getBucketPolicy(getBucketPolicyRequest); + } + + @Override + public void setBucketPolicy(String bucketName, String policyText) throws AmazonClientException, AmazonServiceException { + delegate.setBucketPolicy(bucketName, policyText); + } + + @Override + public void setBucketPolicy(SetBucketPolicyRequest setBucketPolicyRequest) throws AmazonClientException, AmazonServiceException { + delegate.setBucketPolicy(setBucketPolicyRequest); + } + + @Override + public void deleteBucketPolicy(String bucketName) throws AmazonClientException, AmazonServiceException { + delegate.deleteBucketPolicy(bucketName); + } + + @Override + public void deleteBucketPolicy(DeleteBucketPolicyRequest deleteBucketPolicyRequest) throws AmazonClientException, AmazonServiceException { + delegate.deleteBucketPolicy(deleteBucketPolicyRequest); + } + + @Override + public URL generatePresignedUrl(String bucketName, String key, Date expiration) throws AmazonClientException { + return delegate.generatePresignedUrl(bucketName, key, expiration); + } + + @Override + public URL generatePresignedUrl(String bucketName, String key, Date expiration, HttpMethod method) throws AmazonClientException { + return delegate.generatePresignedUrl(bucketName, key, expiration, method); + } + + @Override + public URL generatePresignedUrl(GeneratePresignedUrlRequest generatePresignedUrlRequest) throws AmazonClientException { + return delegate.generatePresignedUrl(generatePresignedUrlRequest); + } + + @Override + public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest request) throws AmazonClientException, AmazonServiceException { + return delegate.initiateMultipartUpload(request); + } + + @Override + public UploadPartResult uploadPart(UploadPartRequest request) throws AmazonClientException, AmazonServiceException { + return delegate.uploadPart(request); + } + + @Override + public PartListing listParts(ListPartsRequest request) throws AmazonClientException, AmazonServiceException { + return delegate.listParts(request); + } + + @Override + public void abortMultipartUpload(AbortMultipartUploadRequest request) throws AmazonClientException, AmazonServiceException { + delegate.abortMultipartUpload(request); + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request) throws AmazonClientException, AmazonServiceException { + return delegate.completeMultipartUpload(request); + } + + @Override + public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest request) throws AmazonClientException, AmazonServiceException { + return delegate.listMultipartUploads(request); + } + + @Override + public S3ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest request) { + return delegate.getCachedResponseMetadata(request); + } + + @Override + public void restoreObject(RestoreObjectRequest copyGlacierObjectRequest) throws AmazonServiceException { + delegate.restoreObject(copyGlacierObjectRequest); + } + + @Override + public void restoreObject(String bucketName, String key, int expirationInDays) throws AmazonServiceException { + delegate.restoreObject(bucketName, key, expirationInDays); + } + + @Override + public void enableRequesterPays(String bucketName) throws AmazonServiceException, AmazonClientException { + delegate.enableRequesterPays(bucketName); + } + + @Override + public void disableRequesterPays(String bucketName) throws AmazonServiceException, AmazonClientException { + delegate.disableRequesterPays(bucketName); + } + + @Override + public boolean isRequesterPaysEnabled(String bucketName) throws AmazonServiceException, AmazonClientException { + return delegate.isRequesterPaysEnabled(bucketName); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java b/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java new file mode 100644 index 00000000..a3c3e02d --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectResult; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble; + +/** + * + */ +public class TestAmazonS3 extends AmazonS3Wrapper { + + private double writeFailureRate = 0.0; + + private String randomPrefix; + + ConcurrentMap accessCounts = new ConcurrentHashMap(); + + private long incrementAndGet(String path) { + AtomicLong value = accessCounts.get(path); + if (value == null) { + value = accessCounts.putIfAbsent(path, new AtomicLong(1)); + } + if (value != null) { + return value.incrementAndGet(); + } + return 1; + } + + public TestAmazonS3(AmazonS3 delegate, Settings componentSettings) { + super(delegate); + randomPrefix = componentSettings.get("test.random"); + writeFailureRate = componentSettings.getAsDouble("test.write_failures", 0.0); + } + + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) throws AmazonClientException, AmazonServiceException { + if (shouldFail(bucketName, key, writeFailureRate)) { + long length = metadata.getContentLength(); + long partToRead = (long) (length * randomDouble()); + byte[] buffer = new byte[1024]; + for (long cur = 0; cur < partToRead; cur += buffer.length) { + try { + input.read(buffer, 0, (int) (partToRead - cur > buffer.length ? buffer.length : partToRead - cur)); + } catch (IOException ex) { + throw new ElasticsearchException("cannot read input stream", ex); + } + } + AmazonS3Exception ex = new AmazonS3Exception("Random S3 exception"); + ex.setStatusCode(400); + ex.setErrorCode("RequestTimeout"); + throw ex; + } else { + return super.putObject(bucketName, key, input, metadata); + } + } + + private boolean shouldFail(String bucketName, String key, double probability) { + if (probability > 0.0) { + String path = randomPrefix + "-" + bucketName + "+" + key; + path += "/" + incrementAndGet(path); + return Math.abs(hashCode(path)) < Integer.MAX_VALUE * probability; + } else { + return false; + } + } + + private int hashCode(String path) { + try { + MessageDigest digest = MessageDigest.getInstance("MD5"); + byte[] bytes = digest.digest(path.getBytes("UTF-8")); + int i = 0; + return ((bytes[i++] & 0xFF) << 24) | ((bytes[i++] & 0xFF) << 16) + | ((bytes[i++] & 0xFF) << 8) | (bytes[i++] & 0xFF); + } catch (UnsupportedEncodingException ex) { + throw new ElasticsearchException("cannot calculate hashcode", ex); + } catch (NoSuchAlgorithmException ex) { + throw new ElasticsearchException("cannot calculate hashcode", ex); + } + } +} diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java new file mode 100644 index 00000000..6372657d --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.aws; + +import com.amazonaws.services.s3.AmazonS3; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; + +import java.util.IdentityHashMap; + +/** + * + */ +public class TestAwsS3Service extends InternalAwsS3Service { + + IdentityHashMap clients = new IdentityHashMap(); + + @Inject + public TestAwsS3Service(Settings settings, SettingsFilter settingsFilter) { + super(settings, settingsFilter); + } + + + @Override + public synchronized AmazonS3 client() { + return cachedWrapper(super.client()); + } + + @Override + public synchronized AmazonS3 client(String region, String account, String key) { + return cachedWrapper(super.client(region, account, key)); + } + + private AmazonS3 cachedWrapper(AmazonS3 client) { + TestAmazonS3 wrapper = clients.get(client); + if (wrapper == null) { + wrapper = new TestAmazonS3(client, componentSettings); + clients.put(client, wrapper); + } + return wrapper; + } + + @Override + protected synchronized void doClose() throws ElasticsearchException { + super.doClose(); + clients.clear(); + } + + +} From c3c9a44d693580b71b7ef3f093d6dc1ddd5cfac0 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 30 Jul 2014 11:11:46 +0200 Subject: [PATCH 26/86] Tests: update to Lucene 4.9.0 Closes #107. --- README.md | 2 +- pom.xml | 5 ++- .../blobstore/S3ImmutableBlobContainer.java | 2 +- .../cloud/aws/AbstractAwsTest.java | 41 ++++++++++--------- .../s3/S3SnapshotRestoreTest.java | 9 ++-- 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 64feb0b9..bfa7ede3 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ Replace all occurrences of `access_key`, `secret_key`, `bucket` and `region` wit To run test: ```sh -mvn -Dtests.aws=true -Des.config=/path/to/config/file/elasticsearch.yml clean test +mvn -Dtests.aws=true -Dtests.config=/path/to/config/file/elasticsearch.yml clean test ``` diff --git a/pom.xml b/pom.xml index ef8350d3..9bfe1e73 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,8 @@ - 1.3.0-SNAPSHOT - 4.8.1 + 1.3.1 + 4.9.0 onerror 1 true @@ -227,6 +227,7 @@ ${tests.weekly} ${tests.slow} ${tests.aws} + ${tests.config} ${tests.awaitsfix} ${tests.slow} ${tests.timeoutSuite} diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java index 9ea97bd5..b011e72a 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java @@ -50,7 +50,7 @@ public void run() { try { ObjectMetadata md = new ObjectMetadata(); if (blobStore.serverSideEncryption()) { - md.setServerSideEncryption(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } md.setContentLength(sizeInBytes); blobStore.client().putObject(blobStore.bucket(), buildKey(blobName), is, md); diff --git a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java index 98c79461..3ea6470c 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java @@ -20,8 +20,11 @@ package org.elasticsearch.cloud.aws; import com.carrotsearch.randomizedtesting.annotations.TestGroup; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.FailedToResolveConfigException; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; @@ -37,22 +40,7 @@ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest { /** * Annotation for tests that require AWS to run. AWS tests are disabled by default. - *

- * To enable test add -Dtests.aws=true -Des.config=/path/to/elasticsearch.yml - *

- * The elasticsearch.yml file should contain the following keys - * 

-     * cloud:
-     *      aws:
-     *          access_key: AKVAIQBF2RECL7FJWGJQ
-     *          secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br
-     *          region: "us-west"
-     *
-     * repositories:
-     *      s3:
-     *          bucket: "bucket_name"
-     *
-     *
+ * Look at README file for details on how to run tests */ @Documented @Inherited @@ -67,12 +55,25 @@ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest { @Override protected Settings nodeSettings(int nodeOrdinal) { - return ImmutableSettings.builder() + ImmutableSettings.Builder settings = ImmutableSettings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) - .put( AwsModule.S3_SERVICE_TYPE_KEY, TestAwsS3Service.class) + .put(AwsModule.S3_SERVICE_TYPE_KEY, TestAwsS3Service.class) .put("cloud.aws.test.random", randomInt()) - .put("cloud.aws.test.write_failures", 0.1) - .build(); + .put("cloud.aws.test.write_failures", 0.1); + + Environment environment = new Environment(); + + // if explicit, just load it and don't load from env + try { + if (Strings.hasText(System.getProperty("tests.config"))) { + settings.loadFromUrl(environment.resolveConfig(System.getProperty("tests.config"))); + } else { + fail("to run integration tests, you need to set -Dtest.aws=true and -Dtests.config=/path/to/elasticsearch.yml"); + } + } catch (FailedToResolveConfigException exception) { + fail("your test configuration file is incorrect: " + System.getProperty("tests.config")); + } + return settings.build(); } } diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index 3988c3dc..8acd9398 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -48,8 +48,7 @@ import java.util.List; import java.util.ArrayList; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.*; /** */ @@ -200,7 +199,7 @@ public void testEncryption() { logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); List summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries(); for (S3ObjectSummary summary : summaries) { - assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getServerSideEncryption(), equalTo("AES256")); + assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256")); } logger.info("--> delete some data"); @@ -387,6 +386,10 @@ public void cleanRepositoryFiles(String basePath) { String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key")); String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key")); String bucketName = bucket.get("bucket"); + + // We check that settings has been set in elasticsearch.yml integration test file + // as described in README + assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(region, accessKey, secretKey); try { ObjectListing prevListing = null; From 4a3929e6de9ed2535b1a6fe5952802cde483a9a2 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 11:17:41 +0200 Subject: [PATCH 27/86] Update maven plugins versions and change compile to 1.7 --- pom.xml | 62 ++++++++++++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 9bfe1e73..e8b810c0 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,6 @@ 1.3.1 4.9.0 onerror - 1 true onerror @@ -51,14 +50,18 @@ - - org.apache.lucene - lucene-test-framework - ${lucene.version} + org.hamcrest + hamcrest-core + 1.3.RC2 + test + + + org.hamcrest + hamcrest-library + 1.3.RC2 test - org.apache.lucene lucene-test-framework @@ -79,14 +82,6 @@ provided - - org.elasticsearch - elasticsearch - ${elasticsearch.version} - test-jar - test - - com.amazonaws aws-java-sdk @@ -113,20 +108,6 @@ 1.4 - - org.hamcrest - hamcrest-core - 1.3.RC2 - test - - - - org.hamcrest - hamcrest-library - 1.3.RC2 - test - - log4j log4j @@ -135,6 +116,14 @@ true + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + test-jar + test + + @@ -148,10 +137,10 @@ org.apache.maven.plugins maven-compiler-plugin - 2.3.2 + 3.0 - 1.6 - 1.6 + 1.7 + 1.7 @@ -187,7 +176,7 @@ - ${tests.jvms} + 1 @@ -235,9 +224,6 @@ ${tests.integration} ${tests.cluster_seed} ${tests.client.ratio} - ${env.ES_TEST_LOCAL} - ${es.node.mode} - ${es.config} ${es.logger.level} true @@ -248,7 +234,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.11 + 2.13 true @@ -256,7 +242,7 @@ org.apache.maven.plugins maven-source-plugin - 2.1.2 + 2.2.1 attach-sources @@ -268,7 +254,7 @@ maven-assembly-plugin - 2.3 + 2.4 false ${project.build.directory}/releases/ From 392ed73f69de62537f6e9ae9d9b68b0aaebaeba8 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 11:26:36 +0200 Subject: [PATCH 28/86] Simplify documentation Closes #106 --- README.md | 23 +- dev-tools/build_release.py | 583 ++++++++++++++++++++-------------- dev-tools/email_template.html | 24 ++ dev-tools/email_template.txt | 23 ++ 4 files changed, 396 insertions(+), 257 deletions(-) create mode 100644 dev-tools/email_template.html create mode 100644 dev-tools/email_template.txt diff --git a/README.md b/README.md index bfa7ede3..785a95a5 100644 --- a/README.md +++ b/README.md @@ -4,27 +4,10 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -In order to install the plugin, run: `bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.1.1`. +## Version 2.3.0-SNAPSHOT for Elasticsearch: 1.3 -where `2.1.1` would be the version applicable to your elasticsearch release, as follows: - -* For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/master). -* For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.3). -* For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.2). -* For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.1). -* For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-1.0). -* For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-cloud-aws/tree/es-0.90). - -SNAPSHOT releases are still in development are _NOT_ available for automatic installation with `bin/plugin`. In such cases -you would be responsible building the plugin and providing the file/url yourself. - -| AWS Cloud Plugin | elasticsearch | Release date | -|----------------------------|---------------------|:------------:| -| 2.3.0-SNAPSHOT | 1.3.0 -> 1.3 | XXXX-XX-XX | - -Please read documentation relative to the version you are using: - -* [2.3.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.3/README.md) +If you are looking for another version documentation, please refer to the +[compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). ## Generic Configuration diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index db834544..c6d4db29 100755 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -6,7 +6,7 @@ # not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on @@ -22,6 +22,7 @@ import argparse import github3 import smtplib +import sys from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -45,10 +46,13 @@ - run prerequisite checks ie. check for S3 credentials available as env variables - detect the version to release from the specified branch (--branch) or the current branch - - creates a release branch & updates pom.xml and README.md to point to a release version rather than a snapshot + - check that github issues related to the version are closed + - creates a version release branch & updates pom.xml to point to a release version rather than a snapshot + - creates a master release branch & updates README.md to point to the latest release version for the given elasticsearch branch - builds the artifacts - - commits the new version and merges the release branch into the source branch - - creates a tag and pushes the commit to the specified origin (--remote) + - commits the new version and merges the version release branch into the source branch + - merges the master release branch into the master branch + - creates a tag and pushes branch and master to the specified origin (--remote) - publishes the releases to sonatype and S3 - send a mail based on github issues fixed by this version @@ -68,18 +72,34 @@ env = os.environ LOG = env.get('ES_RELEASE_LOG', '/tmp/elasticsearch_release.log') -ROOT_DIR = os.path.join(abspath(dirname(__file__)), '../') -README_FILE = ROOT_DIR + 'README.md' -POM_FILE = ROOT_DIR + 'pom.xml' +ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) +README_FILE = ROOT_DIR + '/README.md' +POM_FILE = ROOT_DIR + '/pom.xml' +DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' +########################################################## +# +# Utility methods (log and run) +# +########################################################## +# Log a message def log(msg): log_plain('\n%s' % msg) + +# Purge the log file +def purge_log(): + os.remove(LOG) + + +# Log a message to the LOG file def log_plain(msg): f = open(LOG, mode='ab') f.write(msg.encode('utf-8')) f.close() + +# Run a command and log it def run(command, quiet=False): log('%s: RUN: %s\n' % (datetime.datetime.now(), command)) if os.system('%s >> %s 2>&1' % (command, LOG)): @@ -88,19 +108,25 @@ def run(command, quiet=False): print(msg) raise RuntimeError(msg) +########################################################## +# +# Clean logs and check JAVA and Maven +# +########################################################## try: + purge_log() JAVA_HOME = env['JAVA_HOME'] except KeyError: raise RuntimeError(""" Please set JAVA_HOME in the env before running release tool - On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.6*'`""") + On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.7*'`""") try: - MVN='mvn' + MVN = 'mvn' # make sure mvn3 is used if mvn3 is available # some systems use maven 2 as default run('mvn3 --version', quiet=True) - MVN='mvn3' + MVN = 'mvn3' except RuntimeError: pass @@ -109,34 +135,15 @@ def java_exe(): path = JAVA_HOME return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) -# Returns the hash of the current git HEAD revision -def get_head_hash(): - return os.popen(' git rev-parse --verify HEAD 2>&1').read().strip() - -# Returns the hash of the given tag revision -def get_tag_hash(tag): - return os.popen('git show-ref --tags %s --hash 2>&1' % (tag)).read().strip() - -# Returns the name of the current branch -def get_current_branch(): - return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() +########################################################## +# +# String and file manipulation utils +# +########################################################## # Utility that returns the name of the release branch for a given version -def release_branch(version): - return 'release_branch_%s' % version - -# runs get fetch on the given remote -def fetch(remote): - run('git fetch %s' % remote) - -# Creates a new release branch from the given source branch -# and rebases the source branch from the remote before creating -# the release branch. Note: This fails if the source branch -# doesn't exist on the provided remote. -def create_release_branch(remote, src_branch, release): - run('git checkout %s' % src_branch) - run('git pull --rebase %s %s' % (remote, src_branch)) - run('git checkout -b %s' % (release_branch(release))) +def release_branch(branchsource, version): + return 'release_branch_%s_%s' % (branchsource, version) # Reads the given file and applies the @@ -146,7 +153,7 @@ def create_release_branch(remote, src_branch, release): def process_file(file_path, line_callback): fh, abs_path = tempfile.mkstemp() modified = False - with open(abs_path,'w', encoding='utf-8') as new_file: + with open(abs_path, 'w', encoding='utf-8') as new_file: with open(file_path, encoding='utf-8') as old_file: for line in old_file: new_line = line_callback(line) @@ -164,124 +171,107 @@ def process_file(file_path, line_callback): os.remove(abs_path) return False -# Guess the next snapshot version number (increment second digit) + +# Split a version x.y.z as an array of digits [x,y,z] +def split_version_to_digits(version): + return list(map(int, re.findall(r'\d+', version))) + + +# Guess the next snapshot version number (increment last digit) def guess_snapshot(version): - digits=list(map(int, re.findall(r'\d+', version))) - source='%s.%s' % (digits[0], digits[1]) - destination='%s.%s' % (digits[0], digits[1]+1) + digits = split_version_to_digits(version) + source = '%s.%s.%s' % (digits[0], digits[1], digits[2]) + destination = '%s.%s.%s' % (digits[0], digits[1], digits[2] + 1) return version.replace(source, destination) + +# Guess the anchor in generated documentation +# Looks like this "#version-230-for-elasticsearch-13" +def get_doc_anchor(release, esversion): + plugin_digits = split_version_to_digits(release) + es_digits = split_version_to_digits(esversion) + return '#version-%s%s%s-for-elasticsearch-%s%s' % ( + plugin_digits[0], plugin_digits[1], plugin_digits[2], es_digits[0], es_digits[1]) + + # Moves the pom.xml file from a snapshot to a release def remove_maven_snapshot(pom, release): pattern = '%s-SNAPSHOT' % release replacement = '%s' % release + def callback(line): return line.replace(pattern, replacement) + process_file(pom, callback) -# Moves the README.md file from a snapshot to a release -def remove_version_snapshot(readme_file, release): - pattern = '%s-SNAPSHOT' % release - replacement = '%s ' % release - def callback(line): - return line.replace(pattern, replacement) - process_file(readme_file, callback) # Moves the pom.xml file to the next snapshot def add_maven_snapshot(pom, release, snapshot): pattern = '%s' % release replacement = '%s-SNAPSHOT' % snapshot + def callback(line): return line.replace(pattern, replacement) + process_file(pom, callback) -# Add in README.md file the next snapshot -def add_version_snapshot(readme_file, release, snapshot): - pattern = '| %s ' % release - replacement = '| %s-SNAPSHOT' % snapshot + +# Moves the README.md file from a snapshot to a release version. Doc looks like: +# ## Version 2.5.0-SNAPSHOT for Elasticsearch: 1.x +# It needs to be updated to +# ## Version 2.5.0 for Elasticsearch: 1.x +def update_documentation_in_released_branch(readme_file, release, esversion): + pattern = '## Version (.)+ for Elasticsearch: (.)+' + es_digits = split_version_to_digits(esversion) + replacement = '## Version %s for Elasticsearch: %s.%s\n' % ( + release, es_digits[0], es_digits[1]) + def callback(line): - # If we find pattern, we copy the line and replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement).replace('%s' % (datetime.datetime.now().strftime("%Y-%m-%d")), - 'XXXX-XX-XX')+line + # If we find pattern, we replace its content + if re.search(pattern, line) is not None: + return replacement else: return line + process_file(readme_file, callback) + # Moves the README.md file from a snapshot to a release (documentation link) -def remove_documentation_snapshot(readme_file, repo_url, release, branch): - pattern = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (release, repo_url, branch) - replacement = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) +# We need to find the right branch we are on and update the line +# | es-1.3 | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/es-1.3/#version-240-snapshot-for-elasticsearch-13) | +# | es-1.2 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/v2.3.0/#version-230-snapshot-for-elasticsearch-13) | +def update_documentation_to_released_version(readme_file, repo_url, release, branch, esversion): + pattern = '%s' % branch + replacement = '| %s | %s | [%s](%stree/v%s/%s) |\n' % ( + branch, release, release, repo_url, release, get_doc_anchor(release, esversion)) + def callback(line): # If we find pattern, we replace its content if line.find(pattern) >= 0: - return line.replace(pattern, replacement) + return replacement else: return line - process_file(readme_file, callback) -# Add in README.markdown file the documentation for the next version -def add_documentation_snapshot(readme_file, repo_url, release, snapshot, branch): - pattern = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) - replacement = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (snapshot, repo_url, branch) - def callback(line): - # If we find pattern, we copy the line and replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement)+line - else: - return line process_file(readme_file, callback) -# Set release date in README.md file -def set_date(readme_file): - pattern = 'XXXX-XX-XX' - replacement = '%s' % (datetime.datetime.now().strftime("%Y-%m-%d")) - def callback(line): - return line.replace(pattern, replacement) - process_file(readme_file, callback) # Update installation instructions in README.md file def set_install_instructions(readme_file, artifact_name, release): - pattern = '`bin/plugin -install elasticsearch/%s/.+`' % artifact_name - replacement = '`bin/plugin -install elasticsearch/%s/%s`' % (artifact_name, release) + pattern = 'bin/plugin -install elasticsearch/%s/.+' % artifact_name + replacement = 'bin/plugin -install elasticsearch/%s/%s' % (artifact_name, release) + def callback(line): return re.sub(pattern, replacement, line) - process_file(readme_file, callback) + process_file(readme_file, callback) -# Stages the given files for the next git commit -def add_pending_files(*files): - for file in files: - run('git add %s' % file) - -# Executes a git commit with 'release [version]' as the commit message -def commit_release(artifact_id, release): - run('git commit -m "prepare release %s-%s"' % (artifact_id, release)) - -def commit_snapshot(): - run('git commit -m "prepare for next development iteration"') - -def tag_release(release): - run('git tag -a v%s -m "Tag release version %s"' % (release, release)) - -def run_mvn(*cmd): - for c in cmd: - run('%s; %s -f %s %s' % (java_exe(), MVN, POM_FILE, c)) - -def build_release(run_tests=False, dry_run=True): - target = 'deploy' - if dry_run: - target = 'package' - if run_tests: - run_mvn('clean test') - run_mvn('clean %s -DskipTests' %(target)) # Checks the pom.xml for the release version. 2.0.0-SNAPSHOT # This method fails if the pom file has no SNAPSHOT version set ie. # if the version is already on a release version we fail. # Returns the next version string ie. 0.90.7 def find_release_version(src_branch): - run('git checkout %s' % src_branch) + git_checkout(src_branch) with open(POM_FILE, encoding='utf-8') as file: for line in file: match = re.search(r'(.+)-SNAPSHOT', line) @@ -289,6 +279,7 @@ def find_release_version(src_branch): return match.group(1) raise RuntimeError('Could not find release version in branch %s' % src_branch) + # extract a value from pom.xml def find_from_pom(tag): with open(POM_FILE, encoding='utf-8') as file: @@ -298,13 +289,16 @@ def find_from_pom(tag): return match.group(1) raise RuntimeError('Could not find <%s> in pom.xml file' % (tag)) + +# Get artifacts which have been generated in target/releases def get_artifacts(artifact_id, release): - artifact_path = ROOT_DIR + 'target/releases/%s-%s.zip' % (artifact_id, release) - print(' Path %s' % (artifact_path)) + artifact_path = ROOT_DIR + '/target/releases/%s-%s.zip' % (artifact_id, release) + print(' Path %s' % artifact_path) if not os.path.isfile(artifact_path): - raise RuntimeError('Could not find required artifact at %s' % (artifact_path)) + raise RuntimeError('Could not find required artifact at %s' % artifact_path) return artifact_path + # Generates sha1 for a file # and returns the checksum files as well # as the given files in a list @@ -316,38 +310,11 @@ def generate_checksums(release_file): if os.system('cd %s; shasum %s > %s' % (directory, file, checksum_file)): raise RuntimeError('Failed to generate checksum for file %s' % release_file) - res = res + [os.path.join(directory, checksum_file), release_file] + res += [os.path.join(directory, checksum_file), release_file] return res -def git_merge(src_branch, release_version): - run('git checkout %s' % src_branch) - run('git merge %s' % release_branch(release_version)) -def git_push(remote, src_branch, release_version, dry_run): - if not dry_run: - run('git push %s %s' % (remote, src_branch)) # push the commit - run('git push %s v%s' % (remote, release_version)) # push the tag - else: - print(' dryrun [True] -- skipping push to remote %s' % remote) - -def publish_artifacts(artifacts, base='elasticsearch/elasticsearch', dry_run=True): - location = os.path.dirname(os.path.realpath(__file__)) - for artifact in artifacts: - if dry_run: - print('Skip Uploading %s to Amazon S3 in %s' % (artifact, base)) - else: - print('Uploading %s to Amazon S3' % artifact) - # requires boto to be installed but it is not available on python3k yet so we use a dedicated tool - run('python %s/upload-s3.py --file %s --path %s' % (location, os.path.abspath(artifact), base)) - - -################# -## -## -## Email and Github Management -## -## -################# +# Format a GitHub issue as plain text def format_issues_plain(issues, title='Fix'): response = "" @@ -358,6 +325,8 @@ def format_issues_plain(issues, title='Fix'): return response + +# Format a GitHub issue as html text def format_issues_html(issues, title='Fix'): response = "" @@ -369,6 +338,130 @@ def format_issues_html(issues, title='Fix'): return response + +########################################################## +# +# GIT commands +# +########################################################## +# Returns the hash of the current git HEAD revision +def get_head_hash(): + return os.popen('git rev-parse --verify HEAD 2>&1').read().strip() + + +# Returns the name of the current branch +def get_current_branch(): + return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() + + +# runs get fetch on the given remote +def fetch(remote): + run('git fetch %s' % remote) + + +# Creates a new release branch from the given source branch +# and rebases the source branch from the remote before creating +# the release branch. Note: This fails if the source branch +# doesn't exist on the provided remote. +def create_release_branch(remote, src_branch, release): + git_checkout(src_branch) + run('git pull --rebase %s %s' % (remote, src_branch)) + run('git checkout -b %s' % (release_branch(src_branch, release))) + + +# Stages the given files for the next git commit +def add_pending_files(*files): + for file in files: + run('git add %s' % file) + + +# Executes a git commit with 'release [version]' as the commit message +def commit_release(artifact_id, release): + run('git commit -m "prepare release %s-%s"' % (artifact_id, release)) + + +# Commit documentation changes on the master branch +def commit_master(release): + run('git commit -m "update documentation with release %s"' % release) + + +# Commit next snapshot files +def commit_snapshot(): + run('git commit -m "prepare for next development iteration"') + + +# Put the version tag on on the current commit +def tag_release(release): + run('git tag -a v%s -m "Tag release version %s"' % (release, release)) + + +# Checkout a given branch +def git_checkout(branch): + run('git checkout %s' % branch) + + +# Merge the release branch with the actual branch +def git_merge(src_branch, release_version): + git_checkout(src_branch) + run('git merge %s' % release_branch(src_branch, release_version)) + + +# Push the actual branch and master branch +def git_push(remote, src_branch, release_version, dry_run): + if not dry_run: + run('git push %s %s master' % (remote, src_branch)) # push the commit and the master + run('git push %s v%s' % (remote, release_version)) # push the tag + else: + print(' dryrun [True] -- skipping push to remote %s %s master' % (remote, src_branch)) + + +########################################################## +# +# Maven commands +# +########################################################## +# Run a given maven command +def run_mvn(*cmd): + for c in cmd: + run('%s; %s -f %s %s' % (java_exe(), MVN, POM_FILE, c)) + + +# Run deploy or package depending on dry_run +# Default to run mvn package +# When run_tests=True a first mvn clean test is run +def build_release(run_tests=False, dry_run=True): + target = 'deploy' + tests = '-DskipTests' + if run_tests: + tests = '' + if dry_run: + target = 'package' + run_mvn('clean %s %s' % (target, tests)) + + +########################################################## +# +# Amazon S3 publish commands +# +########################################################## +# Upload files to S3 +def publish_artifacts(artifacts, base='elasticsearch/elasticsearch', dry_run=True): + location = os.path.dirname(os.path.realpath(__file__)) + for artifact in artifacts: + if dry_run: + print('Skip Uploading %s to Amazon S3 in %s' % (artifact, base)) + else: + print('Uploading %s to Amazon S3' % artifact) + # requires boto to be installed but it is not available on python3k yet so we use a dedicated tool + run('python %s/upload-s3.py --file %s --path %s' % (location, os.path.abspath(artifact), base)) + + +########################################################## +# +# Email and Github Management +# +########################################################## +# Create a Github repository instance to access issues def get_github_repository(reponame, login=env.get('GITHUB_LOGIN', None), password=env.get('GITHUB_PASSWORD', None), @@ -382,31 +475,45 @@ def get_github_repository(reponame, return g.repository("elasticsearch", reponame) + # Check if there are some remaining open issues and fails def check_opened_issues(version, repository, reponame): opened_issues = [i for i in repository.iter_issues(state='open', labels='%s' % version)] - if len(opened_issues)>0: - raise NameError('Some issues [%s] are still opened. Check https://github.com/elasticsearch/%s/issues?labels=%s&state=open' - % (len(opened_issues), reponame, version)) + if len(opened_issues) > 0: + raise NameError( + 'Some issues [%s] are still opened. Check https://github.com/elasticsearch/%s/issues?labels=%s&state=open' + % (len(opened_issues), reponame, version)) + # List issues from github: can be done anonymously if you don't # exceed a given number of github API calls per day -# Check if there are some remaining open issues and fails def list_issues(version, repository, severity='bug'): issues = [i for i in repository.iter_issues(state='closed', labels='%s,%s' % (severity, version))] return issues + +def read_email_template(format='html'): + file_name = '%s/email_template.%s' % (DEV_TOOLS_DIR, format) + log('open email template %s' % file_name) + with open(file_name, encoding='utf-8') as template_file: + data = template_file.read() + return data + + +# Read template messages +template_email_html = read_email_template('html') +template_email_txt = read_email_template('txt') + + # Get issues from github and generates a Plain/HTML Multipart email -# And send it if dry_run=False def prepare_email(artifact_id, release_version, repository, artifact_name, artifact_description, project_url, severity_labels_bug='bug', severity_labels_update='update', severity_labels_new='new', severity_labels_doc='doc'): - ## Get bugs from github issues_bug = list_issues(release_version, repository, severity=severity_labels_bug) issues_update = list_issues(release_version, repository, severity=severity_labels_update) @@ -425,7 +532,7 @@ def prepare_email(artifact_id, release_version, repository, html_issues_new = format_issues_html(issues_new, 'New') html_issues_doc = format_issues_html(issues_doc, 'Doc') - if len(issues_bug)+len(issues_update)+len(issues_new)+len(issues_doc) > 0: + if len(issues_bug) + len(issues_update) + len(issues_new) + len(issues_doc) > 0: plain_empty_message = "" html_empty_message = "" @@ -435,76 +542,27 @@ def prepare_email(artifact_id, release_version, repository, msg = MIMEMultipart('alternative') msg['Subject'] = '[ANN] %s %s released' % (artifact_name, release_version) - text = """ -Heya, - - -We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s. - -%(artifact_description)s. - -%(project_url)s - -Release Notes - %(artifact_id)s - Version %(release_version)s - -%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -Issues, Pull requests, Feature requests are warmly welcome on %(artifact_id)s project repository: %(project_url)s -For questions or comments around this plugin, feel free to use elasticsearch mailing list: https://groups.google.com/forum/#!forum/elasticsearch - -Enjoy, - --The Elasticsearch team -""" % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': plain_empty_message, - 'issues_bug': plain_issues_bug, - 'issues_update': plain_issues_update, - 'issues_new': plain_issues_new, - 'issues_doc': plain_issues_doc} - - html = """ - - -

Heya,

- -

We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s

- -
%(artifact_description)s.
- -

Release Notes - Version %(release_version)s

-%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -

Issues, Pull requests, Feature requests are warmly welcome on -%(artifact_id)s project repository!

-

For questions or comments around this plugin, feel free to use elasticsearch -mailing list!

- -

Enjoy,

- -

- The Elasticsearch team

- -""" % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': html_empty_message, - 'issues_bug': html_issues_bug, - 'issues_update': html_issues_update, - 'issues_new': html_issues_new, - 'issues_doc': html_issues_doc} + text = template_email_txt % {'release_version': release_version, + 'artifact_id': artifact_id, + 'artifact_name': artifact_name, + 'artifact_description': artifact_description, + 'project_url': project_url, + 'empty_message': plain_empty_message, + 'issues_bug': plain_issues_bug, + 'issues_update': plain_issues_update, + 'issues_new': plain_issues_new, + 'issues_doc': plain_issues_doc} + + html = template_email_html % {'release_version': release_version, + 'artifact_id': artifact_id, + 'artifact_name': artifact_name, + 'artifact_description': artifact_description, + 'project_url': project_url, + 'empty_message': html_empty_message, + 'issues_bug': html_issues_bug, + 'issues_update': html_issues_update, + 'issues_new': html_issues_new, + 'issues_doc': html_issues_doc} # Record the MIME types of both parts - text/plain and text/html. part1 = MIMEText(text, 'plain') @@ -518,6 +576,7 @@ def prepare_email(artifact_id, release_version, repository, return msg + def send_email(msg, dry_run=True, mail=True, @@ -527,14 +586,16 @@ def send_email(msg, msg['From'] = 'Elasticsearch Team <%s>' % sender msg['To'] = 'Elasticsearch Mailing List <%s>' % to # save mail on disk - with open(ROOT_DIR+'target/email.txt', 'w') as email_file: + with open(ROOT_DIR + '/target/email.txt', 'w') as email_file: email_file.write(msg.as_string()) if mail and not dry_run: s = smtplib.SMTP(smtp_server, 25) s.sendmail(sender, to, msg.as_string()) s.quit() else: - print('generated email: open %starget/email.txt' % ROOT_DIR) + print('generated email: open %s/target/email.txt' % ROOT_DIR) + print(msg.as_string()) + def print_sonatype_notice(): settings = os.path.join(os.path.expanduser('~'), '.m2/settings.xml') @@ -566,13 +627,18 @@ def print_sonatype_notice(): """) + def check_s3_credentials(): if not env.get('AWS_ACCESS_KEY_ID', None) or not env.get('AWS_SECRET_ACCESS_KEY', None): - raise RuntimeError('Could not find "AWS_ACCESS_KEY_ID" / "AWS_SECRET_ACCESS_KEY" in the env variables please export in order to upload to S3') + raise RuntimeError( + 'Could not find "AWS_ACCESS_KEY_ID" / "AWS_SECRET_ACCESS_KEY" in the env variables please export in order to upload to S3') + def check_github_credentials(): if not env.get('GITHUB_KEY', None) and not env.get('GITHUB_LOGIN', None): - log('WARN: Could not find "GITHUB_LOGIN" / "GITHUB_PASSWORD" or "GITHUB_KEY" in the env variables. You could need it.') + log( + 'WARN: Could not find "GITHUB_LOGIN" / "GITHUB_PASSWORD" or "GITHUB_KEY" in the env variables. You could need it.') + def check_email_settings(): if not env.get('MAIL_SENDER', None): @@ -605,6 +671,9 @@ def check_email_settings(): dry_run = args.dryrun mail = args.mail + if src_branch == 'master': + raise RuntimeError('Can not release the master branch. You need to create another branch before a release') + if not dry_run: check_s3_credentials() print('WARNING: dryrun is set to "false" - this will push and publish the release') @@ -645,20 +714,33 @@ def check_email_settings(): if not dry_run: smoke_test_version = release_version - head_hash = get_head_hash() - run_mvn('clean') # clean the env! - create_release_branch(remote, src_branch, release_version) - print(' Created release branch [%s]' % (release_branch(release_version))) + + try: + git_checkout('master') + master_hash = get_head_hash() + git_checkout(src_branch) + version_hash = get_head_hash() + run_mvn('clean') # clean the env! + create_release_branch(remote, 'master', release_version) + print(' Created release branch [%s]' % (release_branch('master', release_version))) + create_release_branch(remote, src_branch, release_version) + print(' Created release branch [%s]' % (release_branch(src_branch, release_version))) + except RuntimeError: + print('Logs:') + with open(LOG, 'r') as log_file: + print(log_file.read()) + sys.exit(-1) + success = False try: + ######################################## + # Start update process in version branch + ######################################## pending_files = [POM_FILE, README_FILE] remove_maven_snapshot(POM_FILE, release_version) - remove_documentation_snapshot(README_FILE, project_url, release_version, src_branch) - remove_version_snapshot(README_FILE, release_version) - set_date(README_FILE) - set_install_instructions(README_FILE, artifact_id, release_version) + update_documentation_in_released_branch(README_FILE, release_version, elasticsearch_version) print(' Done removing snapshot version') - add_pending_files(*pending_files) # expects var args use * to expand + add_pending_files(*pending_files) # expects var args use * to expand commit_release(artifact_id, release_version) print(' Committed release version [%s]' % release_version) print(''.join(['-' for _ in range(80)])) @@ -676,25 +758,39 @@ def check_email_settings(): artifact_and_checksums = generate_checksums(artifact) print(''.join(['-' for _ in range(80)])) + ######################################## + # Start update process in master branch + ######################################## + git_checkout(release_branch('master', release_version)) + update_documentation_to_released_version(README_FILE, project_url, release_version, src_branch, + elasticsearch_version) + set_install_instructions(README_FILE, artifact_id, release_version) + add_pending_files(*pending_files) # expects var args use * to expand + commit_master(release_version) + print('Finish Release -- dry_run: %s' % dry_run) input('Press Enter to continue...') + print(' merge release branch') git_merge(src_branch, release_version) print(' tag') tag_release(release_version) add_maven_snapshot(POM_FILE, release_version, snapshot_version) - add_version_snapshot(README_FILE, release_version, snapshot_version) - add_documentation_snapshot(README_FILE, project_url, release_version, snapshot_version, src_branch) + update_documentation_in_released_branch(README_FILE, '%s-SNAPSHOT' % snapshot_version, elasticsearch_version) add_pending_files(*pending_files) commit_snapshot() + print(' merge master branch') + git_merge('master', release_version) + print(' push to %s %s -- dry_run: %s' % (remote, src_branch, dry_run)) git_push(remote, src_branch, release_version, dry_run) print(' publish artifacts to S3 -- dry_run: %s' % dry_run) publish_artifacts(artifact_and_checksums, base='elasticsearch/%s' % (artifact_id) , dry_run=dry_run) print(' preparing email (from github issues)') msg = prepare_email(artifact_id, release_version, repository, artifact_name, artifact_description, project_url) + input('Press Enter to send email...') print(' sending email -- dry_run: %s, mail: %s' % (dry_run, mail)) send_email(msg, dry_run=dry_run, mail=mail) @@ -710,13 +806,26 @@ def check_email_settings(): success = True finally: if not success: - run('git reset --hard HEAD') - run('git checkout %s' % src_branch) + print('Logs:') + with open(LOG, 'r') as log_file: + print(log_file.read()) + git_checkout('master') + run('git reset --hard %s' % master_hash) + git_checkout(src_branch) + run('git reset --hard %s' % version_hash) + try: + run('git tag -d v%s' % release_version) + except RuntimeError: + pass elif dry_run: print('End of dry_run') input('Press Enter to reset changes...') - - run('git reset --hard %s' % head_hash) + git_checkout('master') + run('git reset --hard %s' % master_hash) + git_checkout(src_branch) + run('git reset --hard %s' % version_hash) run('git tag -d v%s' % release_version) + # we delete this one anyways - run('git branch -D %s' % (release_branch(release_version))) + run('git branch -D %s' % (release_branch('master', release_version))) + run('git branch -D %s' % (release_branch(src_branch, release_version))) diff --git a/dev-tools/email_template.html b/dev-tools/email_template.html new file mode 100644 index 00000000..96d0b4b8 --- /dev/null +++ b/dev-tools/email_template.html @@ -0,0 +1,24 @@ + + +

Heya,

+ +

We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s

+ +
%(artifact_description)s.
+ +

Release Notes - Version %(release_version)s

+%(empty_message)s +%(issues_bug)s +%(issues_update)s +%(issues_new)s +%(issues_doc)s + +

Issues, Pull requests, Feature requests are warmly welcome on + %(artifact_id)s project repository!

+

For questions or comments around this plugin, feel free to use elasticsearch + mailing list!

+ +

Enjoy,

+

- The Elasticsearch team

+ + diff --git a/dev-tools/email_template.txt b/dev-tools/email_template.txt new file mode 100644 index 00000000..1550849c --- /dev/null +++ b/dev-tools/email_template.txt @@ -0,0 +1,23 @@ +Heya, + + +We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s. + +%(artifact_description)s. + +%(project_url)s + +Release Notes - %(artifact_id)s - Version %(release_version)s + +%(empty_message)s +%(issues_bug)s +%(issues_update)s +%(issues_new)s +%(issues_doc)s + +Issues, Pull requests, Feature requests are warmly welcome on %(artifact_id)s project repository: %(project_url)s +For questions or comments around this plugin, feel free to use elasticsearch mailing list: https://groups.google.com/forum/#!forum/elasticsearch + +Enjoy, + +-The Elasticsearch team From 77ab67241ba98f79dd9538537782ac615c13c526 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 30 Jul 2014 13:37:40 +0200 Subject: [PATCH 29/86] Wrong exception thrown when snapshot doesn't exist When a Snapshot does not exist, we should raise a `SnapshotMissingException`. Add also tests for GET/DELETE on non existing repo Closes #86. --- .../blobstore/AbstractS3BlobContainer.java | 8 +++ .../s3/S3SnapshotRestoreTest.java | 57 ++++++++++++++++++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java index 5fd28435..b2dfcda9 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; import org.elasticsearch.common.collect.ImmutableMap; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -81,6 +82,13 @@ public void run() { try { S3Object object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); is = object.getObjectContent(); + } catch (AmazonS3Exception e) { + if (e.getStatusCode() == 404) { + listener.onFailure(new FileNotFoundException(e.getMessage())); + } else { + listener.onFailure(e); + } + return; } catch (Throwable e) { listener.onFailure(e); return; diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java index 8acd9398..0737d184 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java @@ -23,11 +23,11 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; - import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.cloud.aws.AbstractAwsTest; import org.elasticsearch.cloud.aws.AbstractAwsTest.AwsTest; import org.elasticsearch.cloud.aws.AwsS3Service; @@ -37,6 +37,7 @@ import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.snapshots.SnapshotMissingException; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; @@ -45,8 +46,8 @@ import org.junit.Before; import org.junit.Test; -import java.util.List; import java.util.ArrayList; +import java.util.List; import static org.hamcrest.Matchers.*; @@ -317,7 +318,57 @@ public void testRepositoryInRemoteRegion() { assertRepositoryIsOperational(client, "test-repo"); } - private void assertRepositoryIsOperational(Client client, String repository) { + /** + * Test case for issue #86: https://github.com/elasticsearch/elasticsearch-cloud-aws/issues/86 + */ + @Test + public void testNonExistingRepo_86() { + Client client = client(); + logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + logger.info("--> restore non existing snapshot"); + try { + client.admin().cluster().prepareRestoreSnapshot("test-repo", "no-existing-snapshot").setWaitForCompletion(true).execute().actionGet(); + fail("Shouldn't be here"); + } catch (SnapshotMissingException ex) { + // Expected + } + } + + /** + * For issue #86: https://github.com/elasticsearch/elasticsearch-cloud-aws/issues/86 + */ + @Test + public void testGetDeleteNonExistingSnapshot_86() { + ClusterAdminClient client = client().admin().cluster(); + logger.info("--> creating azure repository without any path"); + PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("base_path", basePath) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + try { + client.prepareGetSnapshots("test-repo").addSnapshots("no-existing-snapshot").get(); + fail("Shouldn't be here"); + } catch (SnapshotMissingException ex) { + // Expected + } + + try { + client.prepareDeleteSnapshot("test-repo", "no-existing-snapshot").get(); + fail("Shouldn't be here"); + } catch (SnapshotMissingException ex) { + // Expected + } + } + + private void assertRepositoryIsOperational(Client client, String repository) { createIndex("test-idx-1"); ensureGreen(); From 1cdd523903c28036a55c53b3170023eb9e1b38ed Mon Sep 17 00:00:00 2001 From: bitsofinfo Date: Tue, 5 Aug 2014 12:21:39 +0200 Subject: [PATCH 30/86] Allow https communication per ec2 or s3 service By default all communication w/ AWS services done by this plugin is sent the clear over `http`, overriding amazons own default of https: http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ClientConfiguration.html#getProtocol() One has to set `cloud.aws.protocol` in `elasticsearch.yml` to force SSL. cloud.aws.protocol: https This is not entirely clear to the average user, and should be added to the documentation on both this project's README. Closes #101. (cherry picked from commit 0474a1b) --- README.md | 17 +++++++++ .../cloud/aws/AwsEc2Service.java | 1 + .../cloud/aws/InternalAwsS3Service.java | 1 + ...ava => S3SnapshotRestoreAbstractTest.java} | 2 +- .../s3/S3SnapshotRestoreOverHttpTest.java | 35 +++++++++++++++++++ .../s3/S3SnapshotRestoreOverHttpsTest.java | 35 +++++++++++++++++++ 6 files changed, 90 insertions(+), 1 deletion(-) rename src/test/java/org/elasticsearch/repositories/s3/{S3SnapshotRestoreTest.java => S3SnapshotRestoreAbstractTest.java} (99%) create mode 100644 src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java create mode 100644 src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java diff --git a/README.md b/README.md index 785a95a5..f53b1da5 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,23 @@ cloud: secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br ``` +### Transport security + +By default this plugin uses HTTP for all API calls to AWS endpoints. If you wish to configure HTTPS you can set +`cloud.aws.protocol` in the elasticsearch config. You can optionally override this setting per individual service +via: `cloud.aws.ec2.protocol` or `cloud.aws.s3.protocol`. + +``` +cloud: + aws: + protocol: http + s3: + protocol: https + ec2: + protocol: http + +``` + ### Region The `cloud.aws.region` can be set to a region and will automatically use the relevant settings for both `ec2` and `s3`. The available values are: diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index 69829feb..6499ee0e 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -61,6 +61,7 @@ public synchronized AmazonEC2 client() { ClientConfiguration clientConfiguration = new ClientConfiguration(); String protocol = componentSettings.get("protocol", "http").toLowerCase(); + protocol = componentSettings.get("ec2.protocol", protocol).toLowerCase(); if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); } else if ("https".equals(protocol)) { diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 9cf21580..94ec05c1 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -89,6 +89,7 @@ private synchronized AmazonS3 getClient(String endpoint, String account, String ClientConfiguration clientConfiguration = new ClientConfiguration(); String protocol = componentSettings.get("protocol", "http").toLowerCase(); + protocol = componentSettings.get("s3.protocol", protocol).toLowerCase(); if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); } else if ("https".equals(protocol)) { diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreAbstractTest.java similarity index 99% rename from src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java rename to src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreAbstractTest.java index 0737d184..5aaf6758 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreAbstractTest.java @@ -55,7 +55,7 @@ */ @AwsTest @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) -public class S3SnapshotRestoreTest extends AbstractAwsTest { +abstract public class S3SnapshotRestoreAbstractTest extends AbstractAwsTest { @Override public Settings indexSettings() { diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java new file mode 100644 index 00000000..45222b85 --- /dev/null +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.s3; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +/** + */ +public class S3SnapshotRestoreOverHttpTest extends S3SnapshotRestoreAbstractTest { + @Override + public Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("cloud.aws.s3.protocol", "http"); + return settings.build(); + } +} diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java new file mode 100644 index 00000000..f36b0595 --- /dev/null +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.s3; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +/** + */ +public class S3SnapshotRestoreOverHttpsTest extends S3SnapshotRestoreAbstractTest { + @Override + public Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("cloud.aws.s3.protocol", "https"); + return settings.build(); + } +} From 5cc08166643862038ff9f4bff2fa9c3ba9281264 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 12:28:46 +0200 Subject: [PATCH 31/86] Switch to https communication for Amazon APIs by default We should use `https` by default instead of `http` for communication between elasticsearch and AWS API. Note that it can be modified in case of trouble and fallback to the older setting using `cloud.aws.protocol: http` Closes #109. (cherry picked from commit 610d9a7) --- README.md | 9 ++++----- .../java/org/elasticsearch/cloud/aws/AwsEc2Service.java | 2 +- .../elasticsearch/cloud/aws/InternalAwsS3Service.java | 8 ++++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f53b1da5..3d5f1e97 100644 --- a/README.md +++ b/README.md @@ -24,19 +24,18 @@ cloud: ### Transport security -By default this plugin uses HTTP for all API calls to AWS endpoints. If you wish to configure HTTPS you can set +By default this plugin uses HTTPS for all API calls to AWS endpoints. If you wish to configure HTTP you can set `cloud.aws.protocol` in the elasticsearch config. You can optionally override this setting per individual service via: `cloud.aws.ec2.protocol` or `cloud.aws.s3.protocol`. ``` cloud: aws: - protocol: http + protocol: https s3: - protocol: https - ec2: protocol: http - + ec2: + protocol: https ``` ### Region diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index 6499ee0e..fc509362 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -60,7 +60,7 @@ public synchronized AmazonEC2 client() { } ClientConfiguration clientConfiguration = new ClientConfiguration(); - String protocol = componentSettings.get("protocol", "http").toLowerCase(); + String protocol = componentSettings.get("protocol", "https").toLowerCase(); protocol = componentSettings.get("ec2.protocol", protocol).toLowerCase(); if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 94ec05c1..3759ba96 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -19,9 +19,6 @@ package org.elasticsearch.cloud.aws; -import java.util.HashMap; -import java.util.Map; - import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.*; @@ -36,6 +33,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import java.util.HashMap; +import java.util.Map; + /** * */ @@ -88,7 +88,7 @@ private synchronized AmazonS3 getClient(String endpoint, String account, String } ClientConfiguration clientConfiguration = new ClientConfiguration(); - String protocol = componentSettings.get("protocol", "http").toLowerCase(); + String protocol = componentSettings.get("protocol", "https").toLowerCase(); protocol = componentSettings.get("s3.protocol", protocol).toLowerCase(); if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); From b07fc7c03fbd75c9b699d53a841f533994e1c5af Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 13:01:21 +0200 Subject: [PATCH 32/86] Move log4j properties to xml and remove unused elasticsearch.yml file --- src/test/resources/elasticsearch.yml | 21 -------------- src/test/resources/log4j.properties | 9 ------ src/test/resources/log4j.xml | 42 ++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 30 deletions(-) delete mode 100644 src/test/resources/elasticsearch.yml delete mode 100644 src/test/resources/log4j.properties create mode 100644 src/test/resources/log4j.xml diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml deleted file mode 100644 index a7200fa3..00000000 --- a/src/test/resources/elasticsearch.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Replace this access_key / secret_key and bucket name with your own if you want -# to run tests. -# cloud: -# aws: -# access_key: -# secret_key: -# -#discovery: -# type: ec2 -# -#repositories: -# s3: -# bucket: -# region: -# private-bucket: -# bucket: -# access_key: -# secret_key: -# remote-bucket: -# bucket: -# region: \ No newline at end of file diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties deleted file mode 100644 index 151cfa44..00000000 --- a/src/test/resources/log4j.properties +++ /dev/null @@ -1,9 +0,0 @@ -es.logger.level=INFO -log4j.rootLogger=${es.logger.level}, out - -log4j.appender.out=org.apache.log4j.ConsoleAppender -log4j.appender.out.layout=org.apache.log4j.PatternLayout -log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n -log4j.logger.org.elasticsearch.snapshots=TRACE -log4j.logger.org.elasticsearch.index.snapshots=TRACE -log4j.logger.com.amazonaws=INFO diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml new file mode 100644 index 00000000..052175f3 --- /dev/null +++ b/src/test/resources/log4j.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 75103b7d80a146c4db5ecedf8a3a5fb8b69bae0a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 13:39:42 +0200 Subject: [PATCH 33/86] Fix tests --- ...toreAbstractTest.java => AbstractS3SnapshotRestoreTest.java} | 2 +- .../repositories/s3/S3SnapshotRestoreOverHttpTest.java | 2 +- .../repositories/s3/S3SnapshotRestoreOverHttpsTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/test/java/org/elasticsearch/repositories/s3/{S3SnapshotRestoreAbstractTest.java => AbstractS3SnapshotRestoreTest.java} (99%) diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreAbstractTest.java b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java similarity index 99% rename from src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreAbstractTest.java rename to src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 5aaf6758..65d8ce8a 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreAbstractTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -55,7 +55,7 @@ */ @AwsTest @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) -abstract public class S3SnapshotRestoreAbstractTest extends AbstractAwsTest { +abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest { @Override public Settings indexSettings() { diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java index 45222b85..e33c82f6 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTest.java @@ -24,7 +24,7 @@ /** */ -public class S3SnapshotRestoreOverHttpTest extends S3SnapshotRestoreAbstractTest { +public class S3SnapshotRestoreOverHttpTest extends AbstractS3SnapshotRestoreTest { @Override public Settings nodeSettings(int nodeOrdinal) { ImmutableSettings.Builder settings = ImmutableSettings.builder() diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java index f36b0595..975ff95b 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTest.java @@ -24,7 +24,7 @@ /** */ -public class S3SnapshotRestoreOverHttpsTest extends S3SnapshotRestoreAbstractTest { +public class S3SnapshotRestoreOverHttpsTest extends AbstractS3SnapshotRestoreTest { @Override public Settings nodeSettings(int nodeOrdinal) { ImmutableSettings.Builder settings = ImmutableSettings.builder() From 1cc5b3926698391ead38ad7b4508c026f40647fb Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 13:28:02 +0200 Subject: [PATCH 34/86] Use new release tool for elasticsearch (cherry picked from commit cb393d4) --- .gitignore | 1 + dev-tools/build_release.py | 831 ---------------------------------- dev-tools/email_template.html | 24 - dev-tools/email_template.txt | 23 - dev-tools/release.py | 134 ++++++ dev-tools/upload-s3.py | 67 --- 6 files changed, 135 insertions(+), 945 deletions(-) delete mode 100755 dev-tools/build_release.py delete mode 100644 dev-tools/email_template.html delete mode 100644 dev-tools/email_template.txt create mode 100644 dev-tools/release.py delete mode 100644 dev-tools/upload-s3.py diff --git a/.gitignore b/.gitignore index c42ab2ba..0409d72b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /.settings /.project /.classpath +plugin_tools diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py deleted file mode 100755 index c6d4db29..00000000 --- a/dev-tools/build_release.py +++ /dev/null @@ -1,831 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import re -import tempfile -import shutil -import os -import datetime -import argparse -import github3 -import smtplib -import sys - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -from os.path import dirname, abspath - -""" - This tool builds a release from the a given elasticsearch plugin branch. - In order to execute it go in the top level directory and run: - $ python3 dev_tools/build_release.py --branch master --publish --remote origin - - By default this script runs in 'dry' mode which essentially simulates a release. If the - '--publish' option is set the actual release is done. - If not in 'dry' mode, a mail will be automatically sent to the mailing list. - You can disable it with the option '--disable_mail' - - $ python3 dev_tools/build_release.py --publish --remote origin --disable_mail - - The script takes over almost all - steps necessary for a release from a high level point of view it does the following things: - - - run prerequisite checks ie. check for S3 credentials available as env variables - - detect the version to release from the specified branch (--branch) or the current branch - - check that github issues related to the version are closed - - creates a version release branch & updates pom.xml to point to a release version rather than a snapshot - - creates a master release branch & updates README.md to point to the latest release version for the given elasticsearch branch - - builds the artifacts - - commits the new version and merges the version release branch into the source branch - - merges the master release branch into the master branch - - creates a tag and pushes branch and master to the specified origin (--remote) - - publishes the releases to sonatype and S3 - - send a mail based on github issues fixed by this version - -Once it's done it will print all the remaining steps. - - Prerequisites: - - Python 3k for script execution - - Boto for S3 Upload ($ apt-get install python-boto or pip-3.3 install boto) - - github3 module (pip-3.3 install github3.py) - - S3 keys exported via ENV Variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) - - GITHUB (login/password) or key exported via ENV Variables (GITHUB_LOGIN, GITHUB_PASSWORD or GITHUB_KEY) - (see https://github.com/settings/applications#personal-access-tokens) - Optional: default to no authentication - - SMTP_HOST - Optional: default to localhost - - MAIL_SENDER - Optional: default to 'david@pilato.fr': must be authorized to send emails to elasticsearch mailing list - - MAIL_TO - Optional: default to 'elasticsearch@googlegroups.com' -""" -env = os.environ - -LOG = env.get('ES_RELEASE_LOG', '/tmp/elasticsearch_release.log') -ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) -README_FILE = ROOT_DIR + '/README.md' -POM_FILE = ROOT_DIR + '/pom.xml' -DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' - -########################################################## -# -# Utility methods (log and run) -# -########################################################## -# Log a message -def log(msg): - log_plain('\n%s' % msg) - - -# Purge the log file -def purge_log(): - os.remove(LOG) - - -# Log a message to the LOG file -def log_plain(msg): - f = open(LOG, mode='ab') - f.write(msg.encode('utf-8')) - f.close() - - -# Run a command and log it -def run(command, quiet=False): - log('%s: RUN: %s\n' % (datetime.datetime.now(), command)) - if os.system('%s >> %s 2>&1' % (command, LOG)): - msg = ' FAILED: %s [see log %s]' % (command, LOG) - if not quiet: - print(msg) - raise RuntimeError(msg) - -########################################################## -# -# Clean logs and check JAVA and Maven -# -########################################################## -try: - purge_log() - JAVA_HOME = env['JAVA_HOME'] -except KeyError: - raise RuntimeError(""" - Please set JAVA_HOME in the env before running release tool - On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.7*'`""") - -try: - MVN = 'mvn' - # make sure mvn3 is used if mvn3 is available - # some systems use maven 2 as default - run('mvn3 --version', quiet=True) - MVN = 'mvn3' -except RuntimeError: - pass - - -def java_exe(): - path = JAVA_HOME - return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) - - -########################################################## -# -# String and file manipulation utils -# -########################################################## -# Utility that returns the name of the release branch for a given version -def release_branch(branchsource, version): - return 'release_branch_%s_%s' % (branchsource, version) - - -# Reads the given file and applies the -# callback to it. If the callback changed -# a line the given file is replaced with -# the modified input. -def process_file(file_path, line_callback): - fh, abs_path = tempfile.mkstemp() - modified = False - with open(abs_path, 'w', encoding='utf-8') as new_file: - with open(file_path, encoding='utf-8') as old_file: - for line in old_file: - new_line = line_callback(line) - modified = modified or (new_line != line) - new_file.write(new_line) - os.close(fh) - if modified: - #Remove original file - os.remove(file_path) - #Move new file - shutil.move(abs_path, file_path) - return True - else: - # nothing to do - just remove the tmp file - os.remove(abs_path) - return False - - -# Split a version x.y.z as an array of digits [x,y,z] -def split_version_to_digits(version): - return list(map(int, re.findall(r'\d+', version))) - - -# Guess the next snapshot version number (increment last digit) -def guess_snapshot(version): - digits = split_version_to_digits(version) - source = '%s.%s.%s' % (digits[0], digits[1], digits[2]) - destination = '%s.%s.%s' % (digits[0], digits[1], digits[2] + 1) - return version.replace(source, destination) - - -# Guess the anchor in generated documentation -# Looks like this "#version-230-for-elasticsearch-13" -def get_doc_anchor(release, esversion): - plugin_digits = split_version_to_digits(release) - es_digits = split_version_to_digits(esversion) - return '#version-%s%s%s-for-elasticsearch-%s%s' % ( - plugin_digits[0], plugin_digits[1], plugin_digits[2], es_digits[0], es_digits[1]) - - -# Moves the pom.xml file from a snapshot to a release -def remove_maven_snapshot(pom, release): - pattern = '%s-SNAPSHOT' % release - replacement = '%s' % release - - def callback(line): - return line.replace(pattern, replacement) - - process_file(pom, callback) - - -# Moves the pom.xml file to the next snapshot -def add_maven_snapshot(pom, release, snapshot): - pattern = '%s' % release - replacement = '%s-SNAPSHOT' % snapshot - - def callback(line): - return line.replace(pattern, replacement) - - process_file(pom, callback) - - -# Moves the README.md file from a snapshot to a release version. Doc looks like: -# ## Version 2.5.0-SNAPSHOT for Elasticsearch: 1.x -# It needs to be updated to -# ## Version 2.5.0 for Elasticsearch: 1.x -def update_documentation_in_released_branch(readme_file, release, esversion): - pattern = '## Version (.)+ for Elasticsearch: (.)+' - es_digits = split_version_to_digits(esversion) - replacement = '## Version %s for Elasticsearch: %s.%s\n' % ( - release, es_digits[0], es_digits[1]) - - def callback(line): - # If we find pattern, we replace its content - if re.search(pattern, line) is not None: - return replacement - else: - return line - - process_file(readme_file, callback) - - -# Moves the README.md file from a snapshot to a release (documentation link) -# We need to find the right branch we are on and update the line -# | es-1.3 | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/es-1.3/#version-240-snapshot-for-elasticsearch-13) | -# | es-1.2 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-azure/tree/v2.3.0/#version-230-snapshot-for-elasticsearch-13) | -def update_documentation_to_released_version(readme_file, repo_url, release, branch, esversion): - pattern = '%s' % branch - replacement = '| %s | %s | [%s](%stree/v%s/%s) |\n' % ( - branch, release, release, repo_url, release, get_doc_anchor(release, esversion)) - - def callback(line): - # If we find pattern, we replace its content - if line.find(pattern) >= 0: - return replacement - else: - return line - - process_file(readme_file, callback) - - -# Update installation instructions in README.md file -def set_install_instructions(readme_file, artifact_name, release): - pattern = 'bin/plugin -install elasticsearch/%s/.+' % artifact_name - replacement = 'bin/plugin -install elasticsearch/%s/%s' % (artifact_name, release) - - def callback(line): - return re.sub(pattern, replacement, line) - - process_file(readme_file, callback) - - -# Checks the pom.xml for the release version. 2.0.0-SNAPSHOT -# This method fails if the pom file has no SNAPSHOT version set ie. -# if the version is already on a release version we fail. -# Returns the next version string ie. 0.90.7 -def find_release_version(src_branch): - git_checkout(src_branch) - with open(POM_FILE, encoding='utf-8') as file: - for line in file: - match = re.search(r'(.+)-SNAPSHOT', line) - if match: - return match.group(1) - raise RuntimeError('Could not find release version in branch %s' % src_branch) - - -# extract a value from pom.xml -def find_from_pom(tag): - with open(POM_FILE, encoding='utf-8') as file: - for line in file: - match = re.search(r'<%s>(.+)' % (tag, tag), line) - if match: - return match.group(1) - raise RuntimeError('Could not find <%s> in pom.xml file' % (tag)) - - -# Get artifacts which have been generated in target/releases -def get_artifacts(artifact_id, release): - artifact_path = ROOT_DIR + '/target/releases/%s-%s.zip' % (artifact_id, release) - print(' Path %s' % artifact_path) - if not os.path.isfile(artifact_path): - raise RuntimeError('Could not find required artifact at %s' % artifact_path) - return artifact_path - - -# Generates sha1 for a file -# and returns the checksum files as well -# as the given files in a list -def generate_checksums(release_file): - res = [] - directory = os.path.dirname(release_file) - file = os.path.basename(release_file) - checksum_file = '%s.sha1.txt' % file - - if os.system('cd %s; shasum %s > %s' % (directory, file, checksum_file)): - raise RuntimeError('Failed to generate checksum for file %s' % release_file) - res += [os.path.join(directory, checksum_file), release_file] - return res - - -# Format a GitHub issue as plain text -def format_issues_plain(issues, title='Fix'): - response = "" - - if len(issues) > 0: - response += '%s:\n' % title - for issue in issues: - response += ' * [%s] - %s (%s)\n' % (issue.number, issue.title, issue.html_url) - - return response - - -# Format a GitHub issue as html text -def format_issues_html(issues, title='Fix'): - response = "" - - if len(issues) > 0: - response += '

%s

\n
    \n' % title - for issue in issues: - response += '
  • [%s] - %s\n' % (issue.html_url, issue.number, issue.title) - response += '
\n' - - return response - - -########################################################## -# -# GIT commands -# -########################################################## -# Returns the hash of the current git HEAD revision -def get_head_hash(): - return os.popen('git rev-parse --verify HEAD 2>&1').read().strip() - - -# Returns the name of the current branch -def get_current_branch(): - return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() - - -# runs get fetch on the given remote -def fetch(remote): - run('git fetch %s' % remote) - - -# Creates a new release branch from the given source branch -# and rebases the source branch from the remote before creating -# the release branch. Note: This fails if the source branch -# doesn't exist on the provided remote. -def create_release_branch(remote, src_branch, release): - git_checkout(src_branch) - run('git pull --rebase %s %s' % (remote, src_branch)) - run('git checkout -b %s' % (release_branch(src_branch, release))) - - -# Stages the given files for the next git commit -def add_pending_files(*files): - for file in files: - run('git add %s' % file) - - -# Executes a git commit with 'release [version]' as the commit message -def commit_release(artifact_id, release): - run('git commit -m "prepare release %s-%s"' % (artifact_id, release)) - - -# Commit documentation changes on the master branch -def commit_master(release): - run('git commit -m "update documentation with release %s"' % release) - - -# Commit next snapshot files -def commit_snapshot(): - run('git commit -m "prepare for next development iteration"') - - -# Put the version tag on on the current commit -def tag_release(release): - run('git tag -a v%s -m "Tag release version %s"' % (release, release)) - - -# Checkout a given branch -def git_checkout(branch): - run('git checkout %s' % branch) - - -# Merge the release branch with the actual branch -def git_merge(src_branch, release_version): - git_checkout(src_branch) - run('git merge %s' % release_branch(src_branch, release_version)) - - -# Push the actual branch and master branch -def git_push(remote, src_branch, release_version, dry_run): - if not dry_run: - run('git push %s %s master' % (remote, src_branch)) # push the commit and the master - run('git push %s v%s' % (remote, release_version)) # push the tag - else: - print(' dryrun [True] -- skipping push to remote %s %s master' % (remote, src_branch)) - - -########################################################## -# -# Maven commands -# -########################################################## -# Run a given maven command -def run_mvn(*cmd): - for c in cmd: - run('%s; %s -f %s %s' % (java_exe(), MVN, POM_FILE, c)) - - -# Run deploy or package depending on dry_run -# Default to run mvn package -# When run_tests=True a first mvn clean test is run -def build_release(run_tests=False, dry_run=True): - target = 'deploy' - tests = '-DskipTests' - if run_tests: - tests = '' - if dry_run: - target = 'package' - run_mvn('clean %s %s' % (target, tests)) - - -########################################################## -# -# Amazon S3 publish commands -# -########################################################## -# Upload files to S3 -def publish_artifacts(artifacts, base='elasticsearch/elasticsearch', dry_run=True): - location = os.path.dirname(os.path.realpath(__file__)) - for artifact in artifacts: - if dry_run: - print('Skip Uploading %s to Amazon S3 in %s' % (artifact, base)) - else: - print('Uploading %s to Amazon S3' % artifact) - # requires boto to be installed but it is not available on python3k yet so we use a dedicated tool - run('python %s/upload-s3.py --file %s --path %s' % (location, os.path.abspath(artifact), base)) - - -########################################################## -# -# Email and Github Management -# -########################################################## -# Create a Github repository instance to access issues -def get_github_repository(reponame, - login=env.get('GITHUB_LOGIN', None), - password=env.get('GITHUB_PASSWORD', None), - key=env.get('GITHUB_KEY', None)): - if login: - g = github3.login(login, password) - elif key: - g = github3.login(token=key) - else: - g = github3.GitHub() - - return g.repository("elasticsearch", reponame) - - -# Check if there are some remaining open issues and fails -def check_opened_issues(version, repository, reponame): - opened_issues = [i for i in repository.iter_issues(state='open', labels='%s' % version)] - if len(opened_issues) > 0: - raise NameError( - 'Some issues [%s] are still opened. Check https://github.com/elasticsearch/%s/issues?labels=%s&state=open' - % (len(opened_issues), reponame, version)) - - -# List issues from github: can be done anonymously if you don't -# exceed a given number of github API calls per day -def list_issues(version, - repository, - severity='bug'): - issues = [i for i in repository.iter_issues(state='closed', labels='%s,%s' % (severity, version))] - return issues - - -def read_email_template(format='html'): - file_name = '%s/email_template.%s' % (DEV_TOOLS_DIR, format) - log('open email template %s' % file_name) - with open(file_name, encoding='utf-8') as template_file: - data = template_file.read() - return data - - -# Read template messages -template_email_html = read_email_template('html') -template_email_txt = read_email_template('txt') - - -# Get issues from github and generates a Plain/HTML Multipart email -def prepare_email(artifact_id, release_version, repository, - artifact_name, artifact_description, project_url, - severity_labels_bug='bug', - severity_labels_update='update', - severity_labels_new='new', - severity_labels_doc='doc'): - ## Get bugs from github - issues_bug = list_issues(release_version, repository, severity=severity_labels_bug) - issues_update = list_issues(release_version, repository, severity=severity_labels_update) - issues_new = list_issues(release_version, repository, severity=severity_labels_new) - issues_doc = list_issues(release_version, repository, severity=severity_labels_doc) - - ## Format content to plain text - plain_issues_bug = format_issues_plain(issues_bug, 'Fix') - plain_issues_update = format_issues_plain(issues_update, 'Update') - plain_issues_new = format_issues_plain(issues_new, 'New') - plain_issues_doc = format_issues_plain(issues_doc, 'Doc') - - ## Format content to html - html_issues_bug = format_issues_html(issues_bug, 'Fix') - html_issues_update = format_issues_html(issues_update, 'Update') - html_issues_new = format_issues_html(issues_new, 'New') - html_issues_doc = format_issues_html(issues_doc, 'Doc') - - if len(issues_bug) + len(issues_update) + len(issues_new) + len(issues_doc) > 0: - plain_empty_message = "" - html_empty_message = "" - - else: - plain_empty_message = "No issue listed for this release" - html_empty_message = "

No issue listed for this release

" - - msg = MIMEMultipart('alternative') - msg['Subject'] = '[ANN] %s %s released' % (artifact_name, release_version) - text = template_email_txt % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': plain_empty_message, - 'issues_bug': plain_issues_bug, - 'issues_update': plain_issues_update, - 'issues_new': plain_issues_new, - 'issues_doc': plain_issues_doc} - - html = template_email_html % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': html_empty_message, - 'issues_bug': html_issues_bug, - 'issues_update': html_issues_update, - 'issues_new': html_issues_new, - 'issues_doc': html_issues_doc} - - # Record the MIME types of both parts - text/plain and text/html. - part1 = MIMEText(text, 'plain') - part2 = MIMEText(html, 'html') - - # Attach parts into message container. - # According to RFC 2046, the last part of a multipart message, in this case - # the HTML message, is best and preferred. - msg.attach(part1) - msg.attach(part2) - - return msg - - -def send_email(msg, - dry_run=True, - mail=True, - sender=env.get('MAIL_SENDER'), - to=env.get('MAIL_TO', 'elasticsearch@googlegroups.com'), - smtp_server=env.get('SMTP_SERVER', 'localhost')): - msg['From'] = 'Elasticsearch Team <%s>' % sender - msg['To'] = 'Elasticsearch Mailing List <%s>' % to - # save mail on disk - with open(ROOT_DIR + '/target/email.txt', 'w') as email_file: - email_file.write(msg.as_string()) - if mail and not dry_run: - s = smtplib.SMTP(smtp_server, 25) - s.sendmail(sender, to, msg.as_string()) - s.quit() - else: - print('generated email: open %s/target/email.txt' % ROOT_DIR) - print(msg.as_string()) - - -def print_sonatype_notice(): - settings = os.path.join(os.path.expanduser('~'), '.m2/settings.xml') - if os.path.isfile(settings): - with open(settings, encoding='utf-8') as settings_file: - for line in settings_file: - if line.strip() == 'sonatype-nexus-snapshots': - # moving out - we found the indicator no need to print the warning - return - print(""" - NOTE: No sonatype settings detected, make sure you have configured - your sonatype credentials in '~/.m2/settings.xml': - - - ... - - - sonatype-nexus-snapshots - your-jira-id - your-jira-pwd - - - sonatype-nexus-staging - your-jira-id - your-jira-pwd - - - ... - - """) - - -def check_s3_credentials(): - if not env.get('AWS_ACCESS_KEY_ID', None) or not env.get('AWS_SECRET_ACCESS_KEY', None): - raise RuntimeError( - 'Could not find "AWS_ACCESS_KEY_ID" / "AWS_SECRET_ACCESS_KEY" in the env variables please export in order to upload to S3') - - -def check_github_credentials(): - if not env.get('GITHUB_KEY', None) and not env.get('GITHUB_LOGIN', None): - log( - 'WARN: Could not find "GITHUB_LOGIN" / "GITHUB_PASSWORD" or "GITHUB_KEY" in the env variables. You could need it.') - - -def check_email_settings(): - if not env.get('MAIL_SENDER', None): - raise RuntimeError('Could not find "MAIL_SENDER"') - -# we print a notice if we can not find the relevant infos in the ~/.m2/settings.xml -print_sonatype_notice() - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Plugin Release') - parser.add_argument('--branch', '-b', metavar='master', default=get_current_branch(), - help='The branch to release from. Defaults to the current branch.') - parser.add_argument('--skiptests', '-t', dest='tests', action='store_false', - help='Skips tests before release. Tests are run by default.') - parser.set_defaults(tests=True) - parser.add_argument('--remote', '-r', metavar='origin', default='origin', - help='The remote to push the release commit and tag to. Default is [origin]') - parser.add_argument('--publish', '-p', dest='dryrun', action='store_false', - help='Publishes the release. Disable by default.') - parser.add_argument('--disable_mail', '-dm', dest='mail', action='store_false', - help='Do not send a release email. Email is sent by default.') - - parser.set_defaults(dryrun=True) - parser.set_defaults(mail=True) - args = parser.parse_args() - - src_branch = args.branch - remote = args.remote - run_tests = args.tests - dry_run = args.dryrun - mail = args.mail - - if src_branch == 'master': - raise RuntimeError('Can not release the master branch. You need to create another branch before a release') - - if not dry_run: - check_s3_credentials() - print('WARNING: dryrun is set to "false" - this will push and publish the release') - if mail: - check_email_settings() - print('An email to %s will be sent after the release' - % env.get('MAIL_TO', 'elasticsearch@googlegroups.com')) - input('Press Enter to continue...') - - check_github_credentials() - - print(''.join(['-' for _ in range(80)])) - print('Preparing Release from branch [%s] running tests: [%s] dryrun: [%s]' % (src_branch, run_tests, dry_run)) - print(' JAVA_HOME is [%s]' % JAVA_HOME) - print(' Running with maven command: [%s] ' % (MVN)) - - release_version = find_release_version(src_branch) - artifact_id = find_from_pom('artifactId') - artifact_name = find_from_pom('name') - artifact_description = find_from_pom('description') - project_url = find_from_pom('url') - elasticsearch_version = find_from_pom('elasticsearch.version') - print(' Artifact Id: [%s]' % artifact_id) - print(' Release version: [%s]' % release_version) - print(' Elasticsearch: [%s]' % elasticsearch_version) - if elasticsearch_version.find('-SNAPSHOT') != -1: - raise RuntimeError('Can not release with a SNAPSHOT elasticsearch dependency: %s' % elasticsearch_version) - - # extract snapshot - default_snapshot_version = guess_snapshot(release_version) - snapshot_version = input('Enter next snapshot version [%s]:' % default_snapshot_version) - snapshot_version = snapshot_version or default_snapshot_version - - print(' Next version: [%s-SNAPSHOT]' % snapshot_version) - print(' Artifact Name: [%s]' % artifact_name) - print(' Artifact Description: [%s]' % artifact_description) - print(' Project URL: [%s]' % project_url) - - if not dry_run: - smoke_test_version = release_version - - try: - git_checkout('master') - master_hash = get_head_hash() - git_checkout(src_branch) - version_hash = get_head_hash() - run_mvn('clean') # clean the env! - create_release_branch(remote, 'master', release_version) - print(' Created release branch [%s]' % (release_branch('master', release_version))) - create_release_branch(remote, src_branch, release_version) - print(' Created release branch [%s]' % (release_branch(src_branch, release_version))) - except RuntimeError: - print('Logs:') - with open(LOG, 'r') as log_file: - print(log_file.read()) - sys.exit(-1) - - success = False - try: - ######################################## - # Start update process in version branch - ######################################## - pending_files = [POM_FILE, README_FILE] - remove_maven_snapshot(POM_FILE, release_version) - update_documentation_in_released_branch(README_FILE, release_version, elasticsearch_version) - print(' Done removing snapshot version') - add_pending_files(*pending_files) # expects var args use * to expand - commit_release(artifact_id, release_version) - print(' Committed release version [%s]' % release_version) - print(''.join(['-' for _ in range(80)])) - print('Building Release candidate') - input('Press Enter to continue...') - print(' Checking github issues') - repository = get_github_repository(artifact_id) - check_opened_issues(release_version, repository, artifact_id) - if not dry_run: - print(' Running maven builds now and publish to sonatype - run-tests [%s]' % run_tests) - else: - print(' Running maven builds now run-tests [%s]' % run_tests) - build_release(run_tests=run_tests, dry_run=dry_run) - artifact = get_artifacts(artifact_id, release_version) - artifact_and_checksums = generate_checksums(artifact) - print(''.join(['-' for _ in range(80)])) - - ######################################## - # Start update process in master branch - ######################################## - git_checkout(release_branch('master', release_version)) - update_documentation_to_released_version(README_FILE, project_url, release_version, src_branch, - elasticsearch_version) - set_install_instructions(README_FILE, artifact_id, release_version) - add_pending_files(*pending_files) # expects var args use * to expand - commit_master(release_version) - - print('Finish Release -- dry_run: %s' % dry_run) - input('Press Enter to continue...') - - print(' merge release branch') - git_merge(src_branch, release_version) - print(' tag') - tag_release(release_version) - - add_maven_snapshot(POM_FILE, release_version, snapshot_version) - update_documentation_in_released_branch(README_FILE, '%s-SNAPSHOT' % snapshot_version, elasticsearch_version) - add_pending_files(*pending_files) - commit_snapshot() - - print(' merge master branch') - git_merge('master', release_version) - - print(' push to %s %s -- dry_run: %s' % (remote, src_branch, dry_run)) - git_push(remote, src_branch, release_version, dry_run) - print(' publish artifacts to S3 -- dry_run: %s' % dry_run) - publish_artifacts(artifact_and_checksums, base='elasticsearch/%s' % (artifact_id) , dry_run=dry_run) - print(' preparing email (from github issues)') - msg = prepare_email(artifact_id, release_version, repository, artifact_name, artifact_description, project_url) - input('Press Enter to send email...') - print(' sending email -- dry_run: %s, mail: %s' % (dry_run, mail)) - send_email(msg, dry_run=dry_run, mail=mail) - - pending_msg = """ -Release successful pending steps: - * close and release sonatype repo: https://oss.sonatype.org/ - * check if the release is there https://oss.sonatype.org/content/repositories/releases/org/elasticsearch/%(artifact_id)s/%(version)s - * tweet about the release -""" - print(pending_msg % {'version': release_version, - 'artifact_id': artifact_id, - 'project_url': project_url}) - success = True - finally: - if not success: - print('Logs:') - with open(LOG, 'r') as log_file: - print(log_file.read()) - git_checkout('master') - run('git reset --hard %s' % master_hash) - git_checkout(src_branch) - run('git reset --hard %s' % version_hash) - try: - run('git tag -d v%s' % release_version) - except RuntimeError: - pass - elif dry_run: - print('End of dry_run') - input('Press Enter to reset changes...') - git_checkout('master') - run('git reset --hard %s' % master_hash) - git_checkout(src_branch) - run('git reset --hard %s' % version_hash) - run('git tag -d v%s' % release_version) - - # we delete this one anyways - run('git branch -D %s' % (release_branch('master', release_version))) - run('git branch -D %s' % (release_branch(src_branch, release_version))) diff --git a/dev-tools/email_template.html b/dev-tools/email_template.html deleted file mode 100644 index 96d0b4b8..00000000 --- a/dev-tools/email_template.html +++ /dev/null @@ -1,24 +0,0 @@ - - -

Heya,

- -

We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s

- -
%(artifact_description)s.
- -

Release Notes - Version %(release_version)s

-%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -

Issues, Pull requests, Feature requests are warmly welcome on - %(artifact_id)s project repository!

-

For questions or comments around this plugin, feel free to use elasticsearch - mailing list!

- -

Enjoy,

-

- The Elasticsearch team

- - diff --git a/dev-tools/email_template.txt b/dev-tools/email_template.txt deleted file mode 100644 index 1550849c..00000000 --- a/dev-tools/email_template.txt +++ /dev/null @@ -1,23 +0,0 @@ -Heya, - - -We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s. - -%(artifact_description)s. - -%(project_url)s - -Release Notes - %(artifact_id)s - Version %(release_version)s - -%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -Issues, Pull requests, Feature requests are warmly welcome on %(artifact_id)s project repository: %(project_url)s -For questions or comments around this plugin, feel free to use elasticsearch mailing list: https://groups.google.com/forum/#!forum/elasticsearch - -Enjoy, - --The Elasticsearch team diff --git a/dev-tools/release.py b/dev-tools/release.py new file mode 100644 index 00000000..edcc637d --- /dev/null +++ b/dev-tools/release.py @@ -0,0 +1,134 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import datetime +import os +import shutil +import sys +import time +import urllib +import urllib.request +import zipfile + +from os.path import dirname, abspath + +""" + This tool builds a release from the a given elasticsearch plugin branch. + + It is basically a wrapper on top of launch_release.py which: + + - tries to get a more recent version of launch_release.py in ... + - download it if needed + - launch it passing all arguments to it, like: + + $ python3 dev_tools/release.py --branch master --publish --remote origin + + Important options: + + # Dry run + $ python3 dev_tools/release.py + + # Dry run without tests + python3 dev_tools/release.py --skiptests + + # Release, publish artifacts and announce + $ python3 dev_tools/release.py --publish + + See full documentation in launch_release.py +""" +env = os.environ + +# Change this if the source repository for your scripts is at a different location +SOURCE_REPO = 'elasticsearch/elasticsearch-plugins-script' +# We define that we should download again the script after 1 days +SCRIPT_OBSOLETE_DAYS = 1 +# We ignore in master.zip file the following files +IGNORED_FILES = ['.gitignore', 'README.md'] + + +ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) +TARGET_TOOLS_DIR = ROOT_DIR + '/plugin_tools' +DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' +BUILD_RELEASE_FILENAME = 'release.zip' +BUILD_RELEASE_FILE = TARGET_TOOLS_DIR + '/' + BUILD_RELEASE_FILENAME +SOURCE_URL = 'https://github.com/%s/archive/master.zip' % SOURCE_REPO + +# Download a recent version of the release plugin tool +try: + os.mkdir(TARGET_TOOLS_DIR) + print('directory %s created' % TARGET_TOOLS_DIR) +except FileExistsError: + pass + + +try: + # we check latest update. If we ran an update recently, we + # are not going to check it again + download = True + + try: + last_download_time = datetime.datetime.fromtimestamp(os.path.getmtime(BUILD_RELEASE_FILE)) + if (datetime.datetime.now()-last_download_time).days < SCRIPT_OBSOLETE_DAYS: + download = False + except FileNotFoundError: + pass + + if download: + urllib.request.urlretrieve(SOURCE_URL, BUILD_RELEASE_FILE) + with zipfile.ZipFile(BUILD_RELEASE_FILE) as myzip: + for member in myzip.infolist(): + filename = os.path.basename(member.filename) + # skip directories + if not filename: + continue + if filename in IGNORED_FILES: + continue + + # copy file (taken from zipfile's extract) + source = myzip.open(member.filename) + target = open(os.path.join(TARGET_TOOLS_DIR, filename), "wb") + with source, target: + shutil.copyfileobj(source, target) + # We keep the original date + date_time = time.mktime(member.date_time + (0, 0, -1)) + os.utime(os.path.join(TARGET_TOOLS_DIR, filename), (date_time, date_time)) + print('plugin-tools updated from %s' % SOURCE_URL) +except urllib.error.HTTPError: + pass + + +# Let see if we need to update the release.py script itself +source_time = os.path.getmtime(TARGET_TOOLS_DIR + '/release.py') +repo_time = os.path.getmtime(DEV_TOOLS_DIR + '/release.py') +if source_time > repo_time: + input('release.py needs an update. Press a key to update it...') + shutil.copyfile(TARGET_TOOLS_DIR + '/release.py', DEV_TOOLS_DIR + '/release.py') + +# We can launch the build process +try: + PYTHON = 'python' + # make sure python3 is used if python3 is available + # some systems use python 2 as default + os.system('python3 --version > /dev/null 2>&1') + PYTHON = 'python3' +except RuntimeError: + pass + +release_args = '' +for x in range(1, len(sys.argv)): + release_args += ' ' + sys.argv[x] + +os.system('%s %s/build_release.py %s' % (PYTHON, TARGET_TOOLS_DIR, release_args)) diff --git a/dev-tools/upload-s3.py b/dev-tools/upload-s3.py deleted file mode 100644 index 95ea576e..00000000 --- a/dev-tools/upload-s3.py +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import os -import sys -import argparse -try: - import boto.s3 -except: - raise RuntimeError(""" - S3 upload requires boto to be installed - Use one of: - 'pip install -U boto' - 'apt-get install python-boto' - 'easy_install boto' - """) - -import boto.s3 - - -def list_buckets(conn): - return conn.get_all_buckets() - - -def upload_s3(conn, path, key, file, bucket): - print 'Uploading %s to Amazon S3 bucket %s/%s' % \ - (file, bucket, os.path.join(path, key)) - def percent_cb(complete, total): - sys.stdout.write('.') - sys.stdout.flush() - bucket = conn.create_bucket(bucket) - k = bucket.new_key(os.path.join(path, key)) - k.set_contents_from_filename(file, cb=percent_cb, num_cb=100) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Uploads files to Amazon S3') - parser.add_argument('--file', '-f', metavar='path to file', - help='the branch to release from', required=True) - parser.add_argument('--bucket', '-b', metavar='B42', default='download.elasticsearch.org', - help='The S3 Bucket to upload to') - parser.add_argument('--path', '-p', metavar='elasticsearch/elasticsearch', default='elasticsearch/elasticsearch', - help='The key path to use') - parser.add_argument('--key', '-k', metavar='key', default=None, - help='The key - uses the file name as default key') - args = parser.parse_args() - if args.key: - key = args.key - else: - key = os.path.basename(args.file) - - connection = boto.connect_s3() - upload_s3(connection, args.path, key, args.file, args.bucket); - From c28bf83b2c6ba4dd226e8b905a77fd77e8d78528 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 13:50:34 +0200 Subject: [PATCH 35/86] prepare release elasticsearch-cloud-aws-2.3.0 --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3d5f1e97..34cf9f06 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.3.0-SNAPSHOT for Elasticsearch: 1.3 +## Version 2.3.0 for Elasticsearch: 1.3 If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index e8b810c0..c51eb5f5 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.3.0-SNAPSHOT + 2.3.0 jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 7ae4b21ef806862176d11c57cffff59b3d788bf7 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 13:56:06 +0200 Subject: [PATCH 36/86] prepare for next development iteration --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 34cf9f06..34990b98 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.3.0 for Elasticsearch: 1.3 +## Version 2.3.1-SNAPSHOT for Elasticsearch: 1.3 If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index c51eb5f5..2a0e9f53 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.3.0 + 2.3.1-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 6c83341cfb9a8eed15b9b0ff976e966202c34818 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 4 Sep 2014 06:37:04 +0200 Subject: [PATCH 37/86] Update to elasticsearch 1.4.0 Closes #114. --- README.md | 2 +- pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 34990b98..da0825e1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.3.1-SNAPSHOT for Elasticsearch: 1.3 +## Version 2.4.0-SNAPSHOT for Elasticsearch: 1.4 If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index 2a0e9f53..ca310a65 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.3.1-SNAPSHOT + 2.4.0-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. @@ -32,7 +32,7 @@ - 1.3.1 + 1.4.0-SNAPSHOT 4.9.0 onerror true From 2239b21435383087e2beb1cd9356a31478d5a3b0 Mon Sep 17 00:00:00 2001 From: Jon Dokulil Date: Thu, 4 Sep 2014 06:41:57 +0200 Subject: [PATCH 38/86] [Docs] Include EC2 IAM policy example Closes #112. Closes #113. (cherry picked from commit 13f4be5) --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index da0825e1..e469e15f 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,28 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu * `any_group`: If set to `false`, will require all security groups to be present for the instance to be used for the discovery. Defaults to `true`. * `ping_timeout`: How long to wait for existing EC2 nodes to reply during discovery. Defaults to `3s`. If no unit like `ms`, `s` or `m` is specified, milliseconds are used. +### Recommended EC2 Permissions + +EC2 discovery requires making a call to the EC2 service. You'll want to setup an IAM policy to allow this. You can create a custom policy via the IAM Management Console. It should look similar to this. + +```js +{ + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2014-09-03" +} +``` + + ### Filtering by Tags The ec2 discovery can also filter machines to include in the cluster based on tags (and not just groups). The settings to use include the `discovery.ec2.tag.` prefix. For example, setting `discovery.ec2.tag.stage` to `dev` will only filter instances with a tag key set to `stage`, and a value of `dev`. Several tags set will require all of those tags to be set for the instance to be included. From 8195ab24b175afceec25333c94a315ede24a4d11 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 5 Sep 2014 16:21:04 +0200 Subject: [PATCH 39/86] ZenDiscovery constructor needs ElectMasterService instance Introduced in https://github.com/elasticsearch/elasticsearch/pull/7336 (elasticsearch 1.4 and 2.0), we need to change Ec2Discovery constructor. Closes #115. --- .../java/org/elasticsearch/discovery/ec2/Ec2Discovery.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index 5d85558b..e6a345a9 100755 --- a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.ZenDiscovery; +import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingService; import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; @@ -42,12 +43,12 @@ public class Ec2Discovery extends ZenDiscovery { @Inject - public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, + public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, - DiscoverySettings discoverySettings) { + DiscoverySettings discoverySettings, ElectMasterService electMasterService) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, - discoveryNodeService, pingService, Version.CURRENT, discoverySettings); + discoveryNodeService, pingService, electMasterService, Version.CURRENT, discoverySettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From 856897fe1d6fbd1b91d59f827123af3e2404283e Mon Sep 17 00:00:00 2001 From: tlrx Date: Thu, 9 Oct 2014 16:00:00 +0200 Subject: [PATCH 40/86] Update to Lucene 4.10.1 (cherry picked from commit 82867e3de212ee517a36cdda93c5a3f201f1faec) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ca310a65..da44deee 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 1.4.0-SNAPSHOT - 4.9.0 + 4.10.1 onerror true onerror From 7ba2ebd73085edc95b0a049444c27393997e2141 Mon Sep 17 00:00:00 2001 From: tlrx Date: Thu, 9 Oct 2014 16:02:26 +0200 Subject: [PATCH 41/86] Create branch es-1.4 for elasticsearch 1.4.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index da44deee..10e1b19d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-cloud-aws - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. @@ -32,7 +32,7 @@ - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT 4.10.1 onerror true From 17e6837046d09990c759070342b888f67b8487ab Mon Sep 17 00:00:00 2001 From: tlrx Date: Thu, 9 Oct 2014 16:07:26 +0200 Subject: [PATCH 42/86] Update documentation for branch es-1.x --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e469e15f..7dc114b4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.4.0-SNAPSHOT for Elasticsearch: 1.4 +## Version 2.5.0-SNAPSHOT for Elasticsearch: 1.x If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). From 37bbf778141b4cf7f84360538fc4e7d485d6f47b Mon Sep 17 00:00:00 2001 From: tlrx Date: Thu, 16 Oct 2014 10:24:13 +0200 Subject: [PATCH 43/86] BlobStore: BlobContainer interface changed in elasticsearch 1.4.0 Adding a S3OutputStream that upload blobs to the S3 Storage service with two modes (single/multipart). When the length of the chunk is lower than buffer_size (default to 5mb), the chunk is uploaded with a single request. Otherwise multiple requests are made, each of buffer_size (except the last one which can be lower than buffer_size). For example, when uploading a blob (say, 1Gb) with chunk_size set for accepting large chunks (chunk_size = 5Gb) and buffer_size set to 100Mb, the blob will be sent into 10 multiple parts, each of ~100Mb. Each part upload may failed independently and will be retried 3 times. Closes #117 (cherry picked from commit 4bc4ea6) --- README.md | 1 + pom.xml | 3 +- .../cloud/aws/InternalAwsS3Service.java | 4 + .../aws/blobstore/DefaultS3OutputStream.java | 225 ++++++++++++++++++ ...lobContainer.java => S3BlobContainer.java} | 56 ++--- .../cloud/aws/blobstore/S3BlobStore.java | 35 +-- .../blobstore/S3ImmutableBlobContainer.java | 85 ------- .../cloud/aws/blobstore/S3OutputStream.java | 125 ++++++++++ .../discovery/ec2/Ec2Discovery.java | 3 +- .../repositories/s3/S3Repository.java | 9 +- .../blobstore/MockDefaultS3OutputStream.java | 99 ++++++++ .../aws/blobstore/S3OutputStreamTest.java | 144 +++++++++++ .../s3/AbstractS3SnapshotRestoreTest.java | 17 +- 13 files changed, 649 insertions(+), 157 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java rename src/main/java/org/elasticsearch/cloud/aws/blobstore/{AbstractS3BlobContainer.java => S3BlobContainer.java} (70%) delete mode 100644 src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java create mode 100644 src/main/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStream.java create mode 100644 src/test/java/org/elasticsearch/cloud/aws/blobstore/MockDefaultS3OutputStream.java create mode 100644 src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTest.java diff --git a/README.md b/README.md index 7dc114b4..8b5f6963 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ The following settings are supported: * `chunk_size`: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, i.e. `1g`, `10m`, `5k`. Defaults to `100m`. * `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`. * `server_side_encryption`: When set to `true` files are encrypted on server side using AES256 algorithm. Defaults to `false`. +* `buffer_size`: Minimum threshold below which the chunk is uploaded using a single request. Beyond this threshold, the S3 repository will use the [AWS Multipart Upload API](http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html) to split the chunk into several parts, each of `buffer_size` length, and to upload each part in its own request. Note that positionning a buffer size lower than `5mb` is not allowed since it will prevents the use of the Multipart API and may result in upload errors. Defaults to `5mb`. * `max_retries`: Number of retries in case of S3 errors. Defaults to `3`. The S3 repositories are using the same credentials as the rest of the AWS services provided by this plugin (`discovery`). diff --git a/pom.xml b/pom.xml index 10e1b19d..8dbc75d4 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ 1.5.0-SNAPSHOT 4.10.1 + 1.7.13 onerror true onerror @@ -85,7 +86,7 @@ com.amazonaws aws-java-sdk - 1.7.13 + ${amazonaws.version} compile diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 3759ba96..75efa160 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -22,6 +22,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.*; +import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; @@ -192,5 +193,8 @@ protected void doClose() throws ElasticsearchException { for (AmazonS3Client client : clients.values()) { client.shutdown(); } + + // Ensure that IdleConnectionReaper is shutdown + IdleConnectionReaper.shutdown(); } } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java new file mode 100644 index 00000000..e9c73f78 --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java @@ -0,0 +1,225 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws.blobstore; + +import com.amazonaws.services.s3.model.*; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * DefaultS3OutputStream uploads data to the AWS S3 service using 2 modes: single and multi part. + *

+ * When the length of the chunk is lower than buffer_size, the chunk is uploaded with a single request. + * Otherwise multiple requests are made, each of buffer_size (except the last one which can be lower than buffer_size). + *

+ * Quick facts about S3: + *

+ * Maximum object size: 5 TB + * Maximum number of parts per upload: 10,000 + * Part numbers: 1 to 10,000 (inclusive) + * Part size: 5 MB to 5 GB, last part can be < 5 MB + *

+ * See http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html + * See http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html + */ +public class DefaultS3OutputStream extends S3OutputStream { + + private static final ByteSizeValue MULTIPART_MAX_SIZE = new ByteSizeValue(5, ByteSizeUnit.GB); + + /** + * Multipart Upload API data + */ + private String multipartId; + private int multipartChunks; + private List multiparts; + + public DefaultS3OutputStream(S3BlobStore blobStore, String bucketName, String blobName, int bufferSizeInBytes, int numberOfRetries, boolean serverSideEncryption) { + super(blobStore, bucketName, blobName, bufferSizeInBytes, numberOfRetries, serverSideEncryption); + } + + @Override + public void flush(byte[] bytes, int off, int len, boolean closing) throws IOException { + if (len > MULTIPART_MAX_SIZE.getBytes()) { + throw new IOException("Unable to upload files larger than " + MULTIPART_MAX_SIZE + " to Amazon S3"); + } + + if (!closing) { + if (len < getBufferSize()) { + upload(bytes, off, len); + } else { + if (getFlushCount() == 0) { + initializeMultipart(); + } + uploadMultipart(bytes, off, len, false); + } + } else { + if (multipartId != null) { + uploadMultipart(bytes, off, len, true); + completeMultipart(); + } else { + upload(bytes, off, len); + } + } + } + + /** + * Upload data using a single request. + * + * @param bytes + * @param off + * @param len + * @throws IOException + */ + private void upload(byte[] bytes, int off, int len) throws IOException { + try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) { + int retry = 0; + while (retry < getNumberOfRetries()) { + try { + doUpload(getBlobStore(), getBucketName(), getBlobName(), is, len, isServerSideEncryption()); + break; + } catch (AmazonS3Exception e) { + if (shouldRetry(e)) { + is.reset(); + retry++; + } else { + throw new IOException("Unable to upload object " + getBlobName(), e); + } + } + } + } + } + + protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length, + boolean serverSideEncryption) throws AmazonS3Exception { + ObjectMetadata md = new ObjectMetadata(); + if (serverSideEncryption) { + md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + } + md.setContentLength(length); + blobStore.client().putObject(bucketName, blobName, is, md); + } + + private void initializeMultipart() { + if (multipartId == null) { + multipartId = doInitialize(getBlobStore(), getBucketName(), getBlobName(), isServerSideEncryption()); + if (multipartId != null) { + multipartChunks = 1; + multiparts = new ArrayList<>(); + } + } + } + + protected String doInitialize(S3BlobStore blobStore, String bucketName, String blobName, boolean serverSideEncryption) { + InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, blobName); + if (serverSideEncryption) { + ObjectMetadata md = new ObjectMetadata(); + md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + request.setObjectMetadata(md); + } + return blobStore.client().initiateMultipartUpload(request).getUploadId(); + } + + private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) throws IOException { + try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) { + int retry = 0; + while (retry < getNumberOfRetries()) { + try { + PartETag partETag = doUploadMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, is, len, lastPart); + multiparts.add(partETag); + multipartChunks++; + return; + } catch (AmazonS3Exception e) { + if (shouldRetry(e) && retry < getNumberOfRetries()) { + is.reset(); + retry++; + } else { + abortMultipart(); + throw e; + } + } + } + } + } + + protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is, + int length, boolean lastPart) throws AmazonS3Exception { + UploadPartRequest request = new UploadPartRequest() + .withBucketName(bucketName) + .withKey(blobName) + .withUploadId(uploadId) + .withPartNumber(multipartChunks) + .withInputStream(is) + .withPartSize(length) + .withLastPart(lastPart); + + UploadPartResult response = blobStore.client().uploadPart(request); + return response.getPartETag(); + + } + + private void completeMultipart() { + int retry = 0; + while (retry < getNumberOfRetries()) { + try { + doCompleteMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, multiparts); + multipartId = null; + return; + } catch (AmazonS3Exception e) { + if (shouldRetry(e) && retry < getNumberOfRetries()) { + retry++; + } else { + abortMultipart(); + throw e; + } + } + } + } + + protected void doCompleteMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, List parts) + throws AmazonS3Exception { + CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(bucketName, blobName, uploadId, parts); + blobStore.client().completeMultipartUpload(request); + } + + private void abortMultipart() { + if (multipartId != null) { + try { + doAbortMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId); + } finally { + multipartId = null; + } + } + } + + protected void doAbortMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId) + throws AmazonS3Exception { + blobStore.client().abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, blobName, uploadId)); + } + + protected boolean shouldRetry(AmazonS3Exception e) { + return e.getStatusCode() == 400 && "RequestTimeout".equals(e.getErrorCode()); + } +} diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java similarity index 70% rename from src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java rename to src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index b2dfcda9..d5a10079 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/AbstractS3BlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -23,7 +23,6 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; -import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; @@ -35,17 +34,18 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; /** * */ -public class AbstractS3BlobContainer extends AbstractBlobContainer { +public class S3BlobContainer extends AbstractBlobContainer { protected final S3BlobStore blobStore; protected final String keyPath; - public AbstractS3BlobContainer(BlobPath path, S3BlobStore blobStore) { + public S3BlobContainer(BlobPath path, S3BlobStore blobStore) { super(path); this.blobStore = blobStore; String keyPath = path.buildAsString("/"); @@ -74,38 +74,22 @@ public boolean deleteBlob(String blobName) throws IOException { } @Override - public void readBlob(final String blobName, final ReadBlobListener listener) { - blobStore.executor().execute(new Runnable() { - @Override - public void run() { - InputStream is; - try { - S3Object object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); - is = object.getObjectContent(); - } catch (AmazonS3Exception e) { - if (e.getStatusCode() == 404) { - listener.onFailure(new FileNotFoundException(e.getMessage())); - } else { - listener.onFailure(e); - } - return; - } catch (Throwable e) { - listener.onFailure(e); - return; - } - byte[] buffer = new byte[blobStore.bufferSizeInBytes()]; - try { - int bytesRead; - while ((bytesRead = is.read(buffer)) != -1) { - listener.onPartial(buffer, 0, bytesRead); - } - listener.onCompleted(); - } catch (Throwable e) { - IOUtils.closeWhileHandlingException(is); - listener.onFailure(e); - } + public InputStream openInput(String blobName) throws IOException { + try { + S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); + return s3Object.getObjectContent(); + } catch (AmazonS3Exception e) { + if (e.getStatusCode() == 404) { + throw new FileNotFoundException(e.getMessage()); } - }); + throw e; + } + } + + @Override + public OutputStream createOutput(final String blobName) throws IOException { + // UploadS3OutputStream does buffering internally + return new DefaultS3OutputStream(blobStore, blobStore.bucket(), buildKey(blobName), blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption()); } @Override @@ -145,8 +129,4 @@ protected String buildKey(String blobName) { return keyPath + blobName; } - protected boolean shouldRetry(AmazonS3Exception e) { - return e.getStatusCode() == 400 && "RequestTimeout".equals(e.getErrorCode()); - } - } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index e060fb55..10ce6c73 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -25,46 +25,53 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.blobstore.ImmutableBlobContainer; +import org.elasticsearch.common.blobstore.BlobStoreException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; -import java.util.concurrent.Executor; /** * */ public class S3BlobStore extends AbstractComponent implements BlobStore { + public static final ByteSizeValue MIN_BUFFER_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB); + private final AmazonS3 client; private final String bucket; private final String region; - private final ThreadPool threadPool; - - private final int bufferSizeInBytes; + private final ByteSizeValue bufferSize; private final boolean serverSideEncryption; private final int numberOfRetries; - public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, ThreadPool threadPool, boolean serverSideEncryption) { + + public S3BlobStore(Settings settings, AmazonS3 client, String bucket, String region, boolean serverSideEncryption) { + this(settings, client, bucket, region, serverSideEncryption, null); + } + + public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption, ByteSizeValue bufferSize) { super(settings); this.client = client; this.bucket = bucket; this.region = region; - this.threadPool = threadPool; this.serverSideEncryption = serverSideEncryption; - this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); + this.bufferSize = (bufferSize != null) ? bufferSize : MIN_BUFFER_SIZE; + if (this.bufferSize.getBytes() < MIN_BUFFER_SIZE.getBytes()) { + throw new BlobStoreException("\"Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]"); + } + this.numberOfRetries = settings.getAsInt("max_retries", 3); if (!client.doesBucketExist(bucket)) { if (region != null) { @@ -88,14 +95,10 @@ public String bucket() { return bucket; } - public Executor executor() { - return threadPool.executor(ThreadPool.Names.SNAPSHOT_DATA); - } - public boolean serverSideEncryption() { return serverSideEncryption; } public int bufferSizeInBytes() { - return bufferSizeInBytes; + return bufferSize.bytesAsInt(); } public int numberOfRetries() { @@ -103,8 +106,8 @@ public int numberOfRetries() { } @Override - public ImmutableBlobContainer immutableBlobContainer(BlobPath path) { - return new S3ImmutableBlobContainer(path, this); + public BlobContainer blobContainer(BlobPath path) { + return new S3BlobContainer(path, this); } @Override diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java deleted file mode 100644 index b011e72a..00000000 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3ImmutableBlobContainer.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.cloud.aws.blobstore; - -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ObjectMetadata; -import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.blobstore.ImmutableBlobContainer; -import org.elasticsearch.common.blobstore.support.BlobStores; - -import java.io.IOException; -import java.io.InputStream; - -/** - * - */ -public class S3ImmutableBlobContainer extends AbstractS3BlobContainer implements ImmutableBlobContainer { - - public S3ImmutableBlobContainer(BlobPath path, S3BlobStore blobStore) { - super(path, blobStore); - } - - @Override - public void writeBlob(final String blobName, final InputStream is, final long sizeInBytes, final WriterListener listener) { - blobStore.executor().execute(new Runnable() { - @Override - public void run() { - int retry = 0; - // Read limit is ignored by InputStreamIndexInput, but we will set it anyway in case - // implementation will change - is.mark(Integer.MAX_VALUE); - while (true) { - try { - ObjectMetadata md = new ObjectMetadata(); - if (blobStore.serverSideEncryption()) { - md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); - } - md.setContentLength(sizeInBytes); - blobStore.client().putObject(blobStore.bucket(), buildKey(blobName), is, md); - listener.onCompleted(); - return; - } catch (AmazonS3Exception e) { - if (shouldRetry(e) && retry < blobStore.numberOfRetries()) { - try { - is.reset(); - } catch (IOException ex) { - listener.onFailure(e); - return; - } - retry++; - } else { - listener.onFailure(e); - return; - } - } catch (Throwable e) { - listener.onFailure(e); - return; - } - } - } - }); - } - - @Override - public void writeBlob(String blobName, InputStream is, long sizeInBytes) throws IOException { - BlobStores.syncWriteBlob(this, blobName, is, sizeInBytes); - } -} diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStream.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStream.java new file mode 100644 index 00000000..a1b66ad4 --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStream.java @@ -0,0 +1,125 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws.blobstore; + +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * S3OutputStream buffers data before flushing it to an underlying S3OutputStream. + */ +public abstract class S3OutputStream extends OutputStream { + + /** + * Limit of upload allowed by AWS S3. + */ + protected static final ByteSizeValue MULTIPART_MAX_SIZE = new ByteSizeValue(5, ByteSizeUnit.GB); + protected static final ByteSizeValue MULTIPART_MIN_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB); + + private S3BlobStore blobStore; + private String bucketName; + private String blobName; + private int numberOfRetries; + private boolean serverSideEncryption; + + private byte[] buffer; + private int count; + private long length; + + private int flushCount = 0; + + public S3OutputStream(S3BlobStore blobStore, String bucketName, String blobName, int bufferSizeInBytes, int numberOfRetries, boolean serverSideEncryption) { + this.blobStore = blobStore; + this.bucketName = bucketName; + this.blobName = blobName; + this.numberOfRetries = numberOfRetries; + this.serverSideEncryption = serverSideEncryption; + + if (bufferSizeInBytes < MULTIPART_MIN_SIZE.getBytes()) { + throw new IllegalArgumentException("Buffer size can't be smaller than " + MULTIPART_MIN_SIZE); + } + if (bufferSizeInBytes > MULTIPART_MAX_SIZE.getBytes()) { + throw new IllegalArgumentException("Buffer size can't be larger than " + MULTIPART_MAX_SIZE); + } + + this.buffer = new byte[bufferSizeInBytes]; + } + + public abstract void flush(byte[] bytes, int off, int len, boolean closing) throws IOException; + + private void flushBuffer(boolean closing) throws IOException { + flush(buffer, 0, count, closing); + flushCount++; + count = 0; + } + + @Override + public void write(int b) throws IOException { + if (count >= buffer.length) { + flushBuffer(false); + } + + buffer[count++] = (byte) b; + length++; + } + + @Override + public void close() throws IOException { + if (count > 0) { + flushBuffer(true); + count = 0; + } + } + + public S3BlobStore getBlobStore() { + return blobStore; + } + + public String getBucketName() { + return bucketName; + } + + public String getBlobName() { + return blobName; + } + + public int getBufferSize() { + return buffer.length; + } + + public int getNumberOfRetries() { + return numberOfRetries; + } + + public boolean isServerSideEncryption() { + return serverSideEncryption; + } + + public long getLength() { + return length; + } + + public int getFlushCount() { + return flushCount; + } +} diff --git a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index e6a345a9..7e5e5e11 100755 --- a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -19,7 +19,6 @@ package org.elasticsearch.discovery.ec2; -import org.elasticsearch.Version; import org.elasticsearch.cloud.aws.AwsEc2Service; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; @@ -48,7 +47,7 @@ public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threa DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, DiscoverySettings discoverySettings, ElectMasterService electMasterService) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, - discoveryNodeService, pingService, electMasterService, Version.CURRENT, discoverySettings); + discoveryNodeService, pingService, electMasterService, discoverySettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 48096d32..4948364e 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -32,7 +32,6 @@ import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositorySettings; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; -import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; import java.util.Locale; @@ -72,7 +71,7 @@ public class S3Repository extends BlobStoreRepository { * @throws IOException */ @Inject - public S3Repository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, AwsS3Service s3Service, ThreadPool threadPool) throws IOException { + public S3Repository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, AwsS3Service s3Service) throws IOException { super(name.getName(), repositorySettings, indexShardRepository); String bucket = repositorySettings.settings().get("bucket", componentSettings.get("bucket")); @@ -120,8 +119,9 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, } boolean serverSideEncryption = repositorySettings.settings().getAsBoolean("server_side_encryption", componentSettings.getAsBoolean("server_side_encryption", false)); - logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}]", bucket, region, chunkSize, serverSideEncryption); - blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, threadPool, serverSideEncryption); + ByteSizeValue bufferSize = repositorySettings.settings().getAsBytesSize("buffer_size", componentSettings.getAsBytesSize("buffer_size", null)); + logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}]", bucket, region, chunkSize, serverSideEncryption, bufferSize); + blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, serverSideEncryption, bufferSize); this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); String basePath = repositorySettings.settings().get("base_path", null); @@ -165,5 +165,4 @@ protected ByteSizeValue chunkSize() { return chunkSize; } - } diff --git a/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockDefaultS3OutputStream.java b/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockDefaultS3OutputStream.java new file mode 100644 index 00000000..cd2450f0 --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockDefaultS3OutputStream.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws.blobstore; + +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.PartETag; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import org.elasticsearch.common.io.Streams; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +public class MockDefaultS3OutputStream extends DefaultS3OutputStream { + + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + + private boolean initialized = false; + private boolean completed = false; + private boolean aborted = false; + + private int numberOfUploadRequests = 0; + + public MockDefaultS3OutputStream(int bufferSizeInBytes) { + super(null, "test-bucket", "test-blobname", bufferSizeInBytes, 3, false); + } + + @Override + protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length, boolean serverSideEncryption) throws AmazonS3Exception { + try { + long copied = Streams.copy(is, out); + if (copied != length) { + throw new AmazonS3Exception("Not all the bytes were copied"); + } + numberOfUploadRequests++; + } catch (IOException e) { + throw new AmazonS3Exception(e.getMessage()); + } + } + + @Override + protected String doInitialize(S3BlobStore blobStore, String bucketName, String blobName, boolean serverSideEncryption) { + initialized = true; + return RandomizedTest.randomAsciiOfLength(50); + } + + @Override + protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is, int length, boolean lastPart) throws AmazonS3Exception { + try { + long copied = Streams.copy(is, out); + if (copied != length) { + throw new AmazonS3Exception("Not all the bytes were copied"); + } + return new PartETag(numberOfUploadRequests++, RandomizedTest.randomAsciiOfLength(50)); + } catch (IOException e) { + throw new AmazonS3Exception(e.getMessage()); + } + } + + @Override + protected void doCompleteMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, List parts) throws AmazonS3Exception { + completed = true; + } + + @Override + protected void doAbortMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId) throws AmazonS3Exception { + aborted = true; + } + + public int getNumberOfUploadRequests() { + return numberOfUploadRequests; + } + + public boolean isMultipart() { + return (numberOfUploadRequests > 1) && initialized && completed && !aborted; + } + + public byte[] toByteArray() { + return out.toByteArray(); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTest.java b/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTest.java new file mode 100644 index 00000000..d7081a70 --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws.blobstore; + +import org.elasticsearch.common.base.Charsets; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import static org.elasticsearch.common.io.Streams.copy; +import static org.hamcrest.Matchers.equalTo; + +/** + * Unit test for {@link S3OutputStream}. + */ +public class S3OutputStreamTest extends ElasticsearchTestCase { + + private static final int BUFFER_SIZE = S3BlobStore.MIN_BUFFER_SIZE.bytesAsInt(); + + @Test + public void testWriteLessDataThanBufferSize() throws IOException { + MockDefaultS3OutputStream out = newS3OutputStream(BUFFER_SIZE); + byte[] content = randomUnicodeOfLengthBetween(1, 512).getBytes(Charsets.UTF_8); + copy(content, out); + + // Checks length & content + assertThat(out.getLength(), equalTo((long) content.length)); + assertThat(Arrays.equals(content, out.toByteArray()), equalTo(true)); + + // Checks single/multi part upload + assertThat(out.getBufferSize(), equalTo(BUFFER_SIZE)); + assertThat(out.getFlushCount(), equalTo(1)); + assertThat(out.getNumberOfUploadRequests(), equalTo(1)); + assertFalse(out.isMultipart()); + + } + + @Test + public void testWriteSameDataThanBufferSize() throws IOException { + int size = randomIntBetween(BUFFER_SIZE, 10 * BUFFER_SIZE); + MockDefaultS3OutputStream out = newS3OutputStream(size); + + ByteArrayOutputStream content = new ByteArrayOutputStream(size); + for (int i = 0; i < size; i++) { + content.write(randomByte()); + } + copy(content.toByteArray(), out); + + // Checks length & content + assertThat(out.getLength(), equalTo((long) size)); + assertThat(Arrays.equals(content.toByteArray(), out.toByteArray()), equalTo(true)); + + // Checks single/multi part upload + assertThat(out.getBufferSize(), equalTo(size)); + assertThat(out.getFlushCount(), equalTo(1)); + assertThat(out.getNumberOfUploadRequests(), equalTo(1)); + assertFalse(out.isMultipart()); + + } + + @Test + public void testWriteExactlyNTimesMoreDataThanBufferSize() throws IOException { + int n = randomIntBetween(2, 10); + int length = n * BUFFER_SIZE; + ByteArrayOutputStream content = new ByteArrayOutputStream(length); + + for (int i = 0; i < length; i++) { + content.write(randomByte()); + } + + MockDefaultS3OutputStream out = newS3OutputStream(BUFFER_SIZE); + copy(content.toByteArray(), out); + + // Checks length & content + assertThat(out.getLength(), equalTo((long) length)); + assertThat(Arrays.equals(content.toByteArray(), out.toByteArray()), equalTo(true)); + + // Checks single/multi part upload + assertThat(out.getBufferSize(), equalTo(BUFFER_SIZE)); + assertThat(out.getFlushCount(), equalTo(n)); + + assertThat(out.getNumberOfUploadRequests(), equalTo(n)); + assertTrue(out.isMultipart()); + } + + @Test + public void testWriteRandomNumberOfBytes() throws IOException { + Integer randomBufferSize = randomIntBetween(BUFFER_SIZE, 5 * BUFFER_SIZE); + MockDefaultS3OutputStream out = newS3OutputStream(randomBufferSize); + + Integer randomLength = randomIntBetween(1, 10 * BUFFER_SIZE); + ByteArrayOutputStream content = new ByteArrayOutputStream(randomLength); + for (int i = 0; i < randomLength; i++) { + content.write(randomByte()); + } + + copy(content.toByteArray(), out); + + // Checks length & content + assertThat(out.getLength(), equalTo((long) randomLength)); + assertThat(Arrays.equals(content.toByteArray(), out.toByteArray()), equalTo(true)); + + assertThat(out.getBufferSize(), equalTo(randomBufferSize)); + int times = (int) Math.ceil(randomLength.doubleValue() / randomBufferSize.doubleValue()); + assertThat(out.getFlushCount(), equalTo(times)); + if (times > 1) { + assertTrue(out.isMultipart()); + } else { + assertFalse(out.isMultipart()); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongBufferSize() throws IOException { + Integer randomBufferSize = randomIntBetween(1, 4 * 1024 * 1024); + MockDefaultS3OutputStream out = newS3OutputStream(randomBufferSize); + fail("Buffer size can't be smaller than 5mb"); + } + + private MockDefaultS3OutputStream newS3OutputStream(int bufferSizeInBytes) { + return new MockDefaultS3OutputStream(bufferSizeInBytes); + } + +} diff --git a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 65d8ce8a..b2e59c8b 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -34,9 +34,9 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.RepositoryVerificationException; import org.elasticsearch.snapshots.SnapshotMissingException; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; @@ -247,19 +247,17 @@ public void testEncryption() { * This test verifies that the test configuration is set up in a manner that * does not make the test {@link #testRepositoryWithCustomCredentials()} pointless. */ - @Test(expected = UncategorizedExecutionException.class) + @Test(expected = RepositoryVerificationException.class) public void assertRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredentials() { Client client = client(); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() .put("base_path", basePath) .put("bucket", bucketSettings.get("bucket")) ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - assertRepositoryIsOperational(client, "test-repo"); + fail("repository verification should have raise an exception!"); } @Test @@ -284,21 +282,20 @@ public void testRepositoryWithCustomCredentials() { * This test verifies that the test configuration is set up in a manner that * does not make the test {@link #testRepositoryInRemoteRegion()} pointless. */ - @Test(expected = UncategorizedExecutionException.class) + @Test(expected = RepositoryVerificationException.class) public void assertRepositoryInRemoteRegionIsRemote() { Client client = client(); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + client.admin().cluster().preparePutRepository("test-repo") .setType("s3").setSettings(ImmutableSettings.settingsBuilder() .put("base_path", basePath) .put("bucket", bucketSettings.get("bucket")) // Below setting intentionally omitted to assert bucket is not available in default region. // .put("region", privateBucketSettings.get("region")) ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - assertRepositoryIsOperational(client, "test-repo"); + fail("repository verification should have raise an exception!"); } @Test From 664193048ed9d90d5fcae72d37a0633dc0f0f6b7 Mon Sep 17 00:00:00 2001 From: Ayumi Yu Date: Mon, 20 Oct 2014 10:01:39 +0200 Subject: [PATCH 44/86] Update README IAM policy fix version number Closes #127 (cherry picked from commit a373ae1) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b5f6963..47f6a4bf 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ EC2 discovery requires making a call to the EC2 service. You'll want to setup an ] } ], - "Version": "2014-09-03" + "Version": "2012-10-17" } ``` From f2ee477308ec283259e86cd8953685c06a70ee9c Mon Sep 17 00:00:00 2001 From: tlrx Date: Mon, 20 Oct 2014 10:10:00 +0200 Subject: [PATCH 45/86] [DOCS] Add documentation about proxy settings Closes #110 (cherry picked from commit bfaa608) --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 47f6a4bf..ff6d96b5 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,17 @@ cloud: protocol: https ``` +In addition, a proxy can be configured with the `proxy_host` and `proxy_port` settings (note that protocol can be `http` or `https`): + +``` +cloud: + aws: + protocol: https + proxy_host: proxy1.company.com + proxy_port: 8083 +``` + + ### Region The `cloud.aws.region` can be set to a region and will automatically use the relevant settings for both `ec2` and `s3`. The available values are: From 1056cb9ded82a77ece219c6642b91b90fcc89ab1 Mon Sep 17 00:00:00 2001 From: Jun Ohtani Date: Fri, 31 Oct 2014 14:33:42 +0900 Subject: [PATCH 46/86] Update to Lucene 4.10.2 Closes #130 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8dbc75d4..8c584b16 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 1.5.0-SNAPSHOT - 4.10.1 + 4.10.2 1.7.13 onerror true From 4e9c083b796cf4f4fcebbb9bf75312d71aa895c3 Mon Sep 17 00:00:00 2001 From: Jun Ohtani Date: Fri, 31 Oct 2014 14:32:33 +0900 Subject: [PATCH 47/86] Tests: Fix randomizedtest fail Closes #131 --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 8c584b16..2f07395f 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,12 @@ 1.3.RC2 test + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + 2.1.10 + test + org.apache.lucene lucene-test-framework From 7ccbdced92027ef08ed8452f5103b31c33ef2145 Mon Sep 17 00:00:00 2001 From: tlrx Date: Tue, 4 Nov 2014 10:31:57 +0100 Subject: [PATCH 48/86] Fix constructors of ZenDiscovery's sub classes Closes #133 (cherry picked from commit 338a94dbe650cdc58f6bb6ae1c1308700629606d) Conflicts: src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java --- .../org/elasticsearch/discovery/ec2/Ec2Discovery.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index 7e5e5e11..dd137d0c 100755 --- a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.cluster.settings.DynamicSettings; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -42,12 +43,12 @@ public class Ec2Discovery extends ZenDiscovery { @Inject - public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, + public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, - DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, - DiscoverySettings discoverySettings, ElectMasterService electMasterService) { + DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, DiscoverySettings discoverySettings, + ElectMasterService electMasterService, DynamicSettings dynamicSettings) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, - discoveryNodeService, pingService, electMasterService, discoverySettings); + discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From b361462dd77b43d69f35b8d45c166218042c5a85 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 10 Nov 2014 18:29:28 -0500 Subject: [PATCH 49/86] Upgrade to Lucene 4.10.3-snapshot-1637985 --- pom.xml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 2f07395f..cbc1ede4 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,8 @@ 1.5.0-SNAPSHOT - 4.10.2 + 4.10.3 + 4.10.3-snapshot-1637985 1.7.13 onerror true @@ -44,6 +45,10 @@ + + Lucene Snapshots + https://download.elasticsearch.org/lucenesnapshots/1637985/ + sonatype http://oss.sonatype.org/content/repositories/releases/ @@ -72,7 +77,7 @@ org.apache.lucene lucene-test-framework - ${lucene.version} + ${lucene.maven.version} test @@ -85,7 +90,7 @@ org.apache.lucene lucene-core - ${lucene.version} + ${lucene.maven.version} provided From 8728e2adec573c161bcba6d3078a3da889f28e92 Mon Sep 17 00:00:00 2001 From: tlrx Date: Mon, 10 Nov 2014 12:11:34 +0100 Subject: [PATCH 50/86] Add retry logic for S3 connection errors when restoring snapshots This commit adds a retry logic when reading blobs from S3. It also adds a retry logic when initializing a multipart upload and sets the internal "max retries" parameter of the Amazon S3 client with the same value as the "max_retry" parameter set for the snapshot repository (so in worst cases with the default value set to 3, 3x3=9 attempts will be made). The internal S3 client uses an exponential back off strategy between each connection exception (mainly IOException). Closes elasticsearch/elasticsearch#8280 Closes #140. --- .../elasticsearch/cloud/aws/AwsS3Service.java | 2 + .../cloud/aws/InternalAwsS3Service.java | 18 ++++++-- .../aws/blobstore/DefaultS3OutputStream.java | 42 +++++++++-------- .../cloud/aws/blobstore/S3BlobContainer.java | 27 +++++++---- .../cloud/aws/blobstore/S3BlobStore.java | 22 ++++++--- .../repositories/s3/S3Repository.java | 8 +++- .../cloud/aws/AbstractAwsTest.java | 3 +- .../elasticsearch/cloud/aws/TestAmazonS3.java | 46 +++++++++++++++++-- .../cloud/aws/TestAwsS3Service.java | 5 ++ 9 files changed, 130 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index af014767..fb01a0b9 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -29,4 +29,6 @@ public interface AwsS3Service extends LifecycleComponent { AmazonS3 client(); AmazonS3 client(String region, String account, String key); + + AmazonS3 client(String region, String account, String key, Integer maxRetries); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 75efa160..a5828e40 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -60,11 +60,16 @@ public synchronized AmazonS3 client() { String account = componentSettings.get("access_key", settings.get("cloud.account")); String key = componentSettings.get("secret_key", settings.get("cloud.key")); - return getClient(endpoint, account, key); + return getClient(endpoint, account, key, null); } @Override - public synchronized AmazonS3 client(String region, String account, String key) { + public AmazonS3 client(String region, String account, String key) { + return client(region, account, key, null); + } + + @Override + public synchronized AmazonS3 client(String region, String account, String key, Integer maxRetries) { String endpoint; if (region == null) { endpoint = getDefaultEndpoint(); @@ -77,11 +82,11 @@ public synchronized AmazonS3 client(String region, String account, String key) { key = componentSettings.get("secret_key", settings.get("cloud.key")); } - return getClient(endpoint, account, key); + return getClient(endpoint, account, key, maxRetries); } - private synchronized AmazonS3 getClient(String endpoint, String account, String key) { + private synchronized AmazonS3 getClient(String endpoint, String account, String key, Integer maxRetries) { Tuple clientDescriptor = new Tuple(endpoint, account); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { @@ -111,6 +116,11 @@ private synchronized AmazonS3 getClient(String endpoint, String account, String clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); } + if (maxRetries != null) { + // If not explicitly set, default to 3 with exponential backoff policy + clientConfiguration.setMaxErrorRetry(maxRetries); + } + AWSCredentialsProvider credentials; if (account == null && key == null) { diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java index e9c73f78..04d30f2b 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java @@ -19,6 +19,7 @@ package org.elasticsearch.cloud.aws.blobstore; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.model.*; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -96,12 +97,12 @@ public void flush(byte[] bytes, int off, int len, boolean closing) throws IOExce private void upload(byte[] bytes, int off, int len) throws IOException { try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) { int retry = 0; - while (retry < getNumberOfRetries()) { + while (retry <= getNumberOfRetries()) { try { doUpload(getBlobStore(), getBucketName(), getBlobName(), is, len, isServerSideEncryption()); break; - } catch (AmazonS3Exception e) { - if (shouldRetry(e)) { + } catch (AmazonClientException e) { + if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { is.reset(); retry++; } else { @@ -123,11 +124,20 @@ protected void doUpload(S3BlobStore blobStore, String bucketName, String blobNam } private void initializeMultipart() { - if (multipartId == null) { - multipartId = doInitialize(getBlobStore(), getBucketName(), getBlobName(), isServerSideEncryption()); - if (multipartId != null) { - multipartChunks = 1; - multiparts = new ArrayList<>(); + int retry = 0; + while ((retry <= getNumberOfRetries()) && (multipartId == null)) { + try { + multipartId = doInitialize(getBlobStore(), getBucketName(), getBlobName(), isServerSideEncryption()); + if (multipartId != null) { + multipartChunks = 1; + multiparts = new ArrayList<>(); + } + } catch (AmazonClientException e) { + if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { + retry++; + } else { + throw e; + } } } } @@ -145,14 +155,14 @@ protected String doInitialize(S3BlobStore blobStore, String bucketName, String b private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) throws IOException { try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) { int retry = 0; - while (retry < getNumberOfRetries()) { + while (retry <= getNumberOfRetries()) { try { PartETag partETag = doUploadMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, is, len, lastPart); multiparts.add(partETag); multipartChunks++; return; - } catch (AmazonS3Exception e) { - if (shouldRetry(e) && retry < getNumberOfRetries()) { + } catch (AmazonClientException e) { + if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { is.reset(); retry++; } else { @@ -182,13 +192,13 @@ protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, S private void completeMultipart() { int retry = 0; - while (retry < getNumberOfRetries()) { + while (retry <= getNumberOfRetries()) { try { doCompleteMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, multiparts); multipartId = null; return; - } catch (AmazonS3Exception e) { - if (shouldRetry(e) && retry < getNumberOfRetries()) { + } catch (AmazonClientException e) { + if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { retry++; } else { abortMultipart(); @@ -218,8 +228,4 @@ protected void doAbortMultipart(S3BlobStore blobStore, String bucketName, String throws AmazonS3Exception { blobStore.client().abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, blobName, uploadId)); } - - protected boolean shouldRetry(AmazonS3Exception e) { - return e.getStatusCode() == 400 && "RequestTimeout".equals(e.getErrorCode()); - } } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index d5a10079..94ff64ff 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -19,6 +19,7 @@ package org.elasticsearch.cloud.aws.blobstore; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; @@ -75,20 +76,30 @@ public boolean deleteBlob(String blobName) throws IOException { @Override public InputStream openInput(String blobName) throws IOException { - try { - S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); - return s3Object.getObjectContent(); - } catch (AmazonS3Exception e) { - if (e.getStatusCode() == 404) { - throw new FileNotFoundException(e.getMessage()); + int retry = 0; + while (retry <= blobStore.numberOfRetries()) { + try { + S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); + return s3Object.getObjectContent(); + } catch (AmazonClientException e) { + if (blobStore.shouldRetry(e) && (retry < blobStore.numberOfRetries())) { + retry++; + } else { + if (e instanceof AmazonS3Exception) { + if (404 == ((AmazonS3Exception) e).getStatusCode()) { + throw new FileNotFoundException("Blob object [" + blobName + "] not found: " + e.getMessage()); + } + } + throw e; + } } - throw e; } + throw new BlobStoreException("retries exhausted while attempting to access blob object [name:" + blobName + ", bucket:" + blobStore.bucket() +"]"); } @Override public OutputStream createOutput(final String blobName) throws IOException { - // UploadS3OutputStream does buffering internally + // UploadS3OutputStream does buffering & retry logic internally return new DefaultS3OutputStream(blobStore, blobStore.bucket(), buildKey(blobName), blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption()); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index 10ce6c73..fcb9e67f 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -19,7 +19,9 @@ package org.elasticsearch.cloud.aws.blobstore; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.ObjectListing; @@ -55,12 +57,8 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { private final int numberOfRetries; - - public S3BlobStore(Settings settings, AmazonS3 client, String bucket, String region, boolean serverSideEncryption) { - this(settings, client, bucket, region, serverSideEncryption, null); - } - - public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption, ByteSizeValue bufferSize) { + public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption, + ByteSizeValue bufferSize, int maxRetries) { super(settings); this.client = client; this.bucket = bucket; @@ -72,7 +70,7 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable throw new BlobStoreException("\"Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]"); } - this.numberOfRetries = settings.getAsInt("max_retries", 3); + this.numberOfRetries = maxRetries; if (!client.doesBucketExist(bucket)) { if (region != null) { client.createBucket(bucket, region); @@ -152,6 +150,16 @@ public void delete(BlobPath path) { } } + protected boolean shouldRetry(AmazonClientException e) { + if (e instanceof AmazonS3Exception) { + AmazonS3Exception s3e = (AmazonS3Exception)e; + if (s3e.getStatusCode() == 400 && "RequestTimeout".equals(s3e.getErrorCode())) { + return true; + } + } + return e.isRetryable(); + } + @Override public void close() { } diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 4948364e..f4bcd5bc 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -120,10 +120,14 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, boolean serverSideEncryption = repositorySettings.settings().getAsBoolean("server_side_encryption", componentSettings.getAsBoolean("server_side_encryption", false)); ByteSizeValue bufferSize = repositorySettings.settings().getAsBytesSize("buffer_size", componentSettings.getAsBytesSize("buffer_size", null)); - logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}]", bucket, region, chunkSize, serverSideEncryption, bufferSize); - blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, serverSideEncryption, bufferSize); + Integer maxRetries = repositorySettings.settings().getAsInt("max_retries", componentSettings.getAsInt("max_retries", 3)); this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); + + logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}]", + bucket, region, chunkSize, serverSideEncryption, bufferSize, maxRetries); + + blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key"), maxRetries), bucket, region, serverSideEncryption, bufferSize, maxRetries); String basePath = repositorySettings.settings().get("base_path", null); if (Strings.hasLength(basePath)) { BlobPath path = new BlobPath(); diff --git a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java index 3ea6470c..537fa058 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java @@ -60,7 +60,8 @@ protected Settings nodeSettings(int nodeOrdinal) { .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .put(AwsModule.S3_SERVICE_TYPE_KEY, TestAwsS3Service.class) .put("cloud.aws.test.random", randomInt()) - .put("cloud.aws.test.write_failures", 0.1); + .put("cloud.aws.test.write_failures", 0.1) + .put("cloud.aws.test.read_failures", 0.1); Environment environment = new Environment(); diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java b/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java index a3c3e02d..cdf167be 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java @@ -22,10 +22,10 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.*; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import java.io.IOException; @@ -44,7 +44,10 @@ */ public class TestAmazonS3 extends AmazonS3Wrapper { + protected final ESLogger logger = Loggers.getLogger(getClass()); + private double writeFailureRate = 0.0; + private double readFailureRate = 0.0; private String randomPrefix; @@ -65,6 +68,7 @@ public TestAmazonS3(AmazonS3 delegate, Settings componentSettings) { super(delegate); randomPrefix = componentSettings.get("test.random"); writeFailureRate = componentSettings.getAsDouble("test.write_failures", 0.0); + readFailureRate = componentSettings.getAsDouble("test.read_failures", 0.0); } @Override @@ -80,6 +84,7 @@ public PutObjectResult putObject(String bucketName, String key, InputStream inpu throw new ElasticsearchException("cannot read input stream", ex); } } + logger.info("--> random write failure on putObject method: throwing an exception for [bucket={}, key={}]", bucketName, key); AmazonS3Exception ex = new AmazonS3Exception("Random S3 exception"); ex.setStatusCode(400); ex.setErrorCode("RequestTimeout"); @@ -89,6 +94,41 @@ public PutObjectResult putObject(String bucketName, String key, InputStream inpu } } + @Override + public UploadPartResult uploadPart(UploadPartRequest request) throws AmazonClientException, AmazonServiceException { + if (shouldFail(request.getBucketName(), request.getKey(), writeFailureRate)) { + long length = request.getPartSize(); + long partToRead = (long) (length * randomDouble()); + byte[] buffer = new byte[1024]; + for (long cur = 0; cur < partToRead; cur += buffer.length) { + try (InputStream input = request.getInputStream()){ + input.read(buffer, 0, (int) (partToRead - cur > buffer.length ? buffer.length : partToRead - cur)); + } catch (IOException ex) { + throw new ElasticsearchException("cannot read input stream", ex); + } + } + logger.info("--> random write failure on uploadPart method: throwing an exception for [bucket={}, key={}]", request.getBucketName(), request.getKey()); + AmazonS3Exception ex = new AmazonS3Exception("Random S3 write exception"); + ex.setStatusCode(400); + ex.setErrorCode("RequestTimeout"); + throw ex; + } else { + return super.uploadPart(request); + } + } + + @Override + public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException { + if (shouldFail(bucketName, key, readFailureRate)) { + logger.info("--> random read failure on getObject method: throwing an exception for [bucket={}, key={}]", bucketName, key); + AmazonS3Exception ex = new AmazonS3Exception("Random S3 read exception"); + ex.setStatusCode(404); + throw ex; + } else { + return super.getObject(bucketName, key); + } + } + private boolean shouldFail(String bucketName, String key, double probability) { if (probability > 0.0) { String path = randomPrefix + "-" + bucketName + "+" + key; diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java index 6372657d..8c4707b8 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -49,6 +49,11 @@ public synchronized AmazonS3 client(String region, String account, String key) { return cachedWrapper(super.client(region, account, key)); } + @Override + public synchronized AmazonS3 client(String region, String account, String key, Integer maxRetries) { + return cachedWrapper(super.client(region, account, key, maxRetries)); + } + private AmazonS3 cachedWrapper(AmazonS3 client) { TestAmazonS3 wrapper = clients.get(client); if (wrapper == null) { From b8aa392b179f098700ba344946caa04e03dc79b7 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 24 Nov 2014 15:54:01 +0100 Subject: [PATCH 51/86] AwsEc2UnicastHostsProvider should use version.minimumCompatibilityVersion() The AwsEc2UnicastHostsProvider creates DiscoveryNodes that are used as an initial seed for unicast based discovery. At the moment it uses Version.CURRENT (see [1]) for those DiscoveryNode object, which confuses the backwards compatibility layer to think this nodes are of the latest version. This causes new nodes to fail to join old nodes as the ping serialization goes wrong. Instead we should use version.minimumCompatibilityVersion(). See [2] [1] https://github.com/elasticsearch/elasticsearch-cloud-aws/blob/es-1.x/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java#L165 [2] https://github.com/elasticsearch/elasticsearch/blob/master/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java#L130 Ps. this was reported on the mailing list. See: https://groups.google.com/forum/#!msg/elasticsearch/8pUwFld88tI/7jRuG6hqtbAJ Closes #143. --- .../discovery/ec2/AwsEc2UnicastHostsProvider.java | 7 +++++-- .../java/org/elasticsearch/discovery/ec2/Ec2Discovery.java | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java index 26637941..1722bd77 100644 --- a/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java @@ -59,6 +59,8 @@ private static enum HostType { private final AmazonEC2 client; + private final Version version; + private final boolean bindAnyGroup; private final ImmutableSet groups; @@ -70,10 +72,11 @@ private static enum HostType { private final HostType hostType; @Inject - public AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AmazonEC2 client) { + public AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AmazonEC2 client, Version version) { super(settings); this.transportService = transportService; this.client = client; + this.version = version; this.hostType = HostType.valueOf(componentSettings.get("host_type", "private_ip").toUpperCase()); @@ -162,7 +165,7 @@ public List buildDynamicNodes() { // we only limit to 1 addresses, makes no sense to ping 100 ports for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { logger.trace("adding {}, address {}, transport_address {}", instance.getInstanceId(), address, addresses[i]); - discoNodes.add(new DiscoveryNode("#cloud-" + instance.getInstanceId() + "-" + i, addresses[i], Version.CURRENT)); + discoNodes.add(new DiscoveryNode("#cloud-" + instance.getInstanceId() + "-" + i, addresses[i], version.minimumCompatibilityVersion())); } } catch (Exception e) { logger.warn("failed ot add {}, address {}", e, instance.getInstanceId(), address); diff --git a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index dd137d0c..2d44e1a2 100755 --- a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -19,6 +19,7 @@ package org.elasticsearch.discovery.ec2; +import org.elasticsearch.Version; import org.elasticsearch.cloud.aws.AwsEc2Service; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; @@ -46,7 +47,8 @@ public class Ec2Discovery extends ZenDiscovery { public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, DiscoverySettings discoverySettings, - ElectMasterService electMasterService, DynamicSettings dynamicSettings) { + ElectMasterService electMasterService, DynamicSettings dynamicSettings, + Version version) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); if (settings.getAsBoolean("cloud.enabled", true)) { @@ -62,7 +64,7 @@ public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threa if (unicastZenPing != null) { // update the unicast zen ping to add cloud hosts provider // and, while we are at it, use only it and not the multicast for example - unicastZenPing.addHostsProvider(new AwsEc2UnicastHostsProvider(settings, transportService, ec2Service.client())); + unicastZenPing.addHostsProvider(new AwsEc2UnicastHostsProvider(settings, transportService, ec2Service.client(), version)); pingService.zenPings(ImmutableList.of(unicastZenPing)); } else { logger.warn("failed to apply ec2 unicast discovery, no unicast ping found"); From d62d1deee423d1c18ca29cf88c132c311ed43ddd Mon Sep 17 00:00:00 2001 From: tlrx Date: Wed, 26 Nov 2014 14:32:41 +0100 Subject: [PATCH 52/86] Remove unnecessary settings in test method Fix unit tests and some bugs introduced by #128 Merge branch 'brutasse-feature/per-repo-endpoint' Merge branch 'feature/per-repo-endpoint' of https://github.com/brutasse/elasticsearch-cloud-aws into brutasse-feature/per-repo-endpoint (cherry picked from commit 59bb2a1) --- README.md | 15 ++++++++--- .../elasticsearch/cloud/aws/AwsS3Service.java | 4 +-- .../cloud/aws/InternalAwsS3Service.java | 26 ++++++++++--------- .../repositories/s3/S3Repository.java | 9 ++++--- .../cloud/aws/TestAwsS3Service.java | 8 +++--- .../s3/AbstractS3SnapshotRestoreTest.java | 26 +++++++++++++++++-- 6 files changed, 62 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index ff6d96b5..3d36bda9 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,8 @@ The following settings are supported: * `bucket`: The name of the bucket to be used for snapshots. (Mandatory) * `region`: The region where bucket is located. Defaults to US Standard +* `endpoint`: The endpoint to the S3 API. Defaults to AWS's default S3 endpoint. Note that setting a region overrides the endpoint setting. +* `protocol`: The protocol to use (`http` or `https`). Defaults to value of `cloud.aws.protocol` or `cloud.aws.s3.protocol`. * `base_path`: Specifies the path within bucket to repository data. Defaults to root directory. * `access_key`: The access key to use for authentication. Defaults to value of `cloud.aws.access_key`. * `secret_key`: The secret key to use for authentication. Defaults to value of `cloud.aws.secret_key`. @@ -241,8 +243,10 @@ The bucket needs to exist to register a repository for snapshots. If you did not ### Using other S3 endpoint -If you are using any S3 api compatible service, you can set the endpoint you want to use by setting `cloud.aws.s3.endpoint` -to your URL provider. +If you are using any S3 api compatible service, you can set a global endpoint by setting `cloud.aws.s3.endpoint` +to your URL provider. Note that this setting will be used for all S3 repositories. + +Different `endpoint`, `region` and `protocol` settings can be set on a per-repository basis (see [S3 Repository](#s3-repository) section for detail). ## Testing @@ -266,7 +270,12 @@ repositories: remote-bucket: bucket: region: - + external-bucket: + bucket: + access_key: + secret_key: + endpoint: + protocol: ``` Replace all occurrences of `access_key`, `secret_key`, `bucket` and `region` with your settings. Please, note that the test will delete all snapshot/restore related files in the specified buckets. diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index fb01a0b9..e5db2ed7 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -28,7 +28,7 @@ public interface AwsS3Service extends LifecycleComponent { AmazonS3 client(); - AmazonS3 client(String region, String account, String key); + AmazonS3 client(String endpoint, String protocol, String region, String account, String key); - AmazonS3 client(String region, String account, String key, Integer maxRetries); + AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index a5828e40..e2f9d17e 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -60,33 +60,32 @@ public synchronized AmazonS3 client() { String account = componentSettings.get("access_key", settings.get("cloud.account")); String key = componentSettings.get("secret_key", settings.get("cloud.key")); - return getClient(endpoint, account, key, null); + return getClient(endpoint, null, account, key, null); } @Override - public AmazonS3 client(String region, String account, String key) { - return client(region, account, key, null); + public AmazonS3 client(String endpoint, String protocol, String region, String account, String key) { + return client(endpoint, protocol, region, account, key, null); } @Override - public synchronized AmazonS3 client(String region, String account, String key, Integer maxRetries) { - String endpoint; - if (region == null) { - endpoint = getDefaultEndpoint(); - } else { + public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) { + if (region != null && endpoint == null) { endpoint = getEndpoint(region); logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + } else if (endpoint == null) { + endpoint = getDefaultEndpoint(); } if (account == null || key == null) { account = componentSettings.get("access_key", settings.get("cloud.account")); key = componentSettings.get("secret_key", settings.get("cloud.key")); } - return getClient(endpoint, account, key, maxRetries); + return getClient(endpoint, protocol, account, key, maxRetries); } - private synchronized AmazonS3 getClient(String endpoint, String account, String key, Integer maxRetries) { + private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries) { Tuple clientDescriptor = new Tuple(endpoint, account); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { @@ -94,8 +93,11 @@ private synchronized AmazonS3 getClient(String endpoint, String account, String } ClientConfiguration clientConfiguration = new ClientConfiguration(); - String protocol = componentSettings.get("protocol", "https").toLowerCase(); - protocol = componentSettings.get("s3.protocol", protocol).toLowerCase(); + if (protocol == null) { + protocol = componentSettings.get("protocol", "https").toLowerCase(); + protocol = componentSettings.get("s3.protocol", protocol).toLowerCase(); + } + if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); } else if ("https".equals(protocol)) { diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index f4bcd5bc..7632576d 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -79,6 +79,9 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, throw new RepositoryException(name.name(), "No bucket defined for s3 gateway"); } + String endpoint = repositorySettings.settings().get("endpoint", componentSettings.get("endpoint")); + String protocol = repositorySettings.settings().get("protocol", componentSettings.get("protocol")); + String region = repositorySettings.settings().get("region", componentSettings.get("region")); if (region == null) { // Bucket setting is not set - use global region setting @@ -124,10 +127,10 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); - logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}]", - bucket, region, chunkSize, serverSideEncryption, bufferSize, maxRetries); + logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}]", + bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries); - blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key"), maxRetries), bucket, region, serverSideEncryption, bufferSize, maxRetries); + blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key"), maxRetries), bucket, region, serverSideEncryption, bufferSize, maxRetries); String basePath = repositorySettings.settings().get("base_path", null); if (Strings.hasLength(basePath)) { BlobPath path = new BlobPath(); diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java index 8c4707b8..b8c9b2ee 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -45,13 +45,13 @@ public synchronized AmazonS3 client() { } @Override - public synchronized AmazonS3 client(String region, String account, String key) { - return cachedWrapper(super.client(region, account, key)); + public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key) { + return cachedWrapper(super.client(endpoint, protocol, region, account, key)); } @Override - public synchronized AmazonS3 client(String region, String account, String key, Integer maxRetries) { - return cachedWrapper(super.client(region, account, key, maxRetries)); + public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) { + return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries)); } private AmazonS3 cachedWrapper(AmazonS3 client) { diff --git a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index b2e59c8b..4cab5316 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -192,6 +192,8 @@ public void testEncryption() { Settings settings = internalCluster().getInstance(Settings.class); Settings bucket = settings.getByPrefix("repositories.s3."); AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( + null, + null, bucket.get("region", settings.get("repositories.s3.region")), bucket.get("access_key", settings.get("cloud.aws.access_key")), bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); @@ -278,6 +280,23 @@ public void testRepositoryWithCustomCredentials() { assertRepositoryIsOperational(client, "test-repo"); } + @Test + public void testRepositoryWithCustomEndpointProtocol() { + Client client = client(); + Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.external-bucket."); + logger.info("--> creating s3 repostoriy with endpoint [{}], bucket[{}] and path [{}]", bucketSettings.get("endpoint"), bucketSettings.get("bucket"), basePath); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(ImmutableSettings.settingsBuilder() + .put("bucket", bucketSettings.get("bucket")) + .put("endpoint", bucketSettings.get("endpoint")) + .put("access_key", bucketSettings.get("access_key")) + .put("secret_key", bucketSettings.get("secret_key")) + .put("base_path", basePath) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + assertRepositoryIsOperational(client, "test-repo"); + } + /** * This test verifies that the test configuration is set up in a manner that * does not make the test {@link #testRepositoryInRemoteRegion()} pointless. @@ -427,9 +446,12 @@ public void cleanRepositoryFiles(String basePath) { Settings[] buckets = { settings.getByPrefix("repositories.s3."), settings.getByPrefix("repositories.s3.private-bucket."), - settings.getByPrefix("repositories.s3.remote-bucket.") + settings.getByPrefix("repositories.s3.remote-bucket."), + settings.getByPrefix("repositories.s3.external-bucket.") }; for (Settings bucket : buckets) { + String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint")); + String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol")); String region = bucket.get("region", settings.get("repositories.s3.region")); String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key")); String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key")); @@ -438,7 +460,7 @@ public void cleanRepositoryFiles(String basePath) { // We check that settings has been set in elasticsearch.yml integration test file // as described in README assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); - AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(region, accessKey, secretKey); + AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey); try { ObjectListing prevListing = null; //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html From 78b11eabaadb890ffc6287daca0d4f113f6918d1 Mon Sep 17 00:00:00 2001 From: tlrx Date: Wed, 26 Nov 2014 17:48:34 +0100 Subject: [PATCH 53/86] Update AWS SDK to 1.9.8 Closes #148 Merge #137 Closes #136 --- README.md | 1 + pom.xml | 21 +++++------- .../cloud/aws/AwsEc2Service.java | 2 ++ .../cloud/aws/InternalAwsS3Service.java | 4 +++ .../repositories/s3/S3Repository.java | 4 +++ .../cloud/aws/AbstractAwsTest.java | 34 +++++++++++++++++++ 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3d36bda9..5b203f7f 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ The `cloud.aws.region` can be set to a region and will automatically use the rel * `ap-southeast-2` * `ap-northeast` (`ap-northeast-1`) * `eu-west` (`eu-west-1`) +* `eu-central` (`eu-central-1`) * `sa-east` (`sa-east-1`). diff --git a/pom.xml b/pom.xml index cbc1ede4..20113cb2 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 1.5.0-SNAPSHOT 4.10.3 4.10.3-snapshot-1637985 - 1.7.13 + 1.9.8 onerror true onerror @@ -96,20 +96,15 @@ com.amazonaws - aws-java-sdk + aws-java-sdk-ec2 + ${amazonaws.version} + compile + + + com.amazonaws + aws-java-sdk-s3 ${amazonaws.version} compile - - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index fc509362..dd15e608 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -121,6 +121,8 @@ public synchronized AmazonEC2 client() { endpoint = "ec2.ap-northeast-1.amazonaws.com"; } else if (region.equals("eu-west") || region.equals("eu-west-1")) { endpoint = "ec2.eu-west-1.amazonaws.com"; + } else if (region.equals("eu-central") || region.equals("eu-central-1")) { + endpoint = "ec2.eu-central-1.amazonaws.com"; } else if (region.equals("sa-east") || region.equals("sa-east-1")) { endpoint = "ec2.sa-east-1.amazonaws.com"; } else { diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index e2f9d17e..e436c107 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -183,6 +183,10 @@ private static String getEndpoint(String region) { return "s3-eu-west-1.amazonaws.com"; } else if ("eu-west-1".equals(region)) { return "s3-eu-west-1.amazonaws.com"; + } else if ("eu-central".equals(region)) { + return "s3.eu-central-1.amazonaws.com"; + } else if ("eu-central-1".equals(region)) { + return "s3.eu-central-1.amazonaws.com"; } else if ("sa-east".equals(region)) { return "s3-sa-east-1.amazonaws.com"; } else if ("sa-east-1".equals(region)) { diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 7632576d..6b19a550 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -113,6 +113,10 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, region = "EU"; } else if ("eu-west-1".equals(regionSetting)) { region = "EU"; + } else if ("eu-central".equals(regionSetting)) { + region = "eu-central-1"; + } else if ("eu-central-1".equals(regionSetting)) { + region = "eu-central-1"; } else if ("sa-east".equals(regionSetting)) { region = "sa-east-1"; } else if ("sa-east-1".equals(regionSetting)) { diff --git a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java index 537fa058..6cbcd44a 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java @@ -27,17 +27,51 @@ import org.elasticsearch.env.FailedToResolveConfigException; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.After; +import org.junit.Before; import java.lang.annotation.Documented; import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.HashMap; +import java.util.Map; /** * */ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest { + /** + * Those properties are set by the AWS SDK v1.9.4 and if not ignored, + * lead to tests failure (see AbstractRandomizedTest#IGNORED_INVARIANT_PROPERTIES) + */ + private static final String[] AWS_INVARIANT_PROPERTIES = { + "com.sun.org.apache.xml.internal.dtm.DTMManager", + "javax.xml.parsers.DocumentBuilderFactory" + }; + + private Map properties = new HashMap<>(); + + @Before + public void saveProperties() { + for (String p : AWS_INVARIANT_PROPERTIES) { + properties.put(p, System.getProperty(p)); + } + } + + @After + public void restoreProperties() { + for (String p : AWS_INVARIANT_PROPERTIES) { + if (properties.get(p) != null) { + System.setProperty(p, properties.get(p)); + } else { + System.clearProperty(p); + } + } + } + + /** * Annotation for tests that require AWS to run. AWS tests are disabled by default. * Look at README file for details on how to run tests From 1518eb9289b161b5b404fbf4a1ee369edf7674a5 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 30 Dec 2014 21:40:47 -0500 Subject: [PATCH 54/86] Upgrade to Lucene 4.10.3 release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 20113cb2..59086d0d 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 1.5.0-SNAPSHOT 4.10.3 - 4.10.3-snapshot-1637985 + 4.10.3 1.9.8 onerror true From 6428d99378f71d06a34186e1934adb0ca71e4681 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 17 Dec 2014 10:18:59 +0100 Subject: [PATCH 55/86] Depend on elasticsearch-parent To simplify plugins maintenance and provide more value in the future, we are starting to build an `elasticsearch-parent` project. This commit is the first step for this plugin to depend on this new `pom` maven project. (cherry picked from commit 2df8350) --- .gitignore | 7 ++-- dev-tools/tests.policy | 54 ++++++++++++++++++++++++++++ pom.xml | 82 +++++++----------------------------------- 3 files changed, 70 insertions(+), 73 deletions(-) create mode 100644 dev-tools/tests.policy diff --git a/.gitignore b/.gitignore index 0409d72b..9533848e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,10 @@ /.idea /target .DS_Store -.local-execution-hints.log *.iml -/.settings /.project +/.settings /.classpath -plugin_tools +/plugin_tools +/.local-execution-hints.log +/.local-*-execution-hints.log diff --git a/dev-tools/tests.policy b/dev-tools/tests.policy new file mode 100644 index 00000000..6afb5025 --- /dev/null +++ b/dev-tools/tests.policy @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Policy file to prevent tests from writing outside the test sandbox directory +// PLEASE NOTE: You may need to enable other permissions when new tests are added, +// everything not allowed here is forbidden! + +grant { + // permissions for file access, write access only to sandbox: + permission java.io.FilePermission "<>", "read,execute"; + permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write"; + permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete"; + permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete"; + permission groovy.security.GroovyCodeSourcePermission "/groovy/script"; + + // Allow connecting to the internet anywhere + permission java.net.SocketPermission "*", "accept,listen,connect,resolve"; + + // Basic permissions needed for Lucene / Elasticsearch to work: + permission java.util.PropertyPermission "*", "read,write"; + permission java.lang.reflect.ReflectPermission "*"; + permission java.lang.RuntimePermission "*"; + + // These two *have* to be spelled out a separate + permission java.lang.management.ManagementPermission "control"; + permission java.lang.management.ManagementPermission "monitor"; + + permission java.net.NetPermission "*"; + permission java.util.logging.LoggingPermission "control"; + permission javax.management.MBeanPermission "*", "*"; + permission javax.management.MBeanServerPermission "*"; + permission javax.management.MBeanTrustPermission "*"; + + // Needed for some things in DNS caching in the JVM + permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl"; + permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl"; + +}; diff --git a/pom.xml b/pom.xml index 59086d0d..12c60d0f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,6 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + + org.elasticsearch + elasticsearch-parent + 1.5.0-SNAPSHOT + + org.elasticsearch elasticsearch-cloud-aws 2.5.0-SNAPSHOT @@ -20,73 +27,33 @@ scm:git:git@github.com:elasticsearch/elasticsearch-cloud-aws.git - scm:git:git@github.com:elasticsearch/elasticsearch-cloud-aws.git - + scm:git:git@github.com:elasticsearch/elasticsearch-cloud-aws.git http://github.com/elasticsearch/elasticsearch-cloud-aws - - org.sonatype.oss - oss-parent - 7 - - - 1.5.0-SNAPSHOT - 4.10.3 - 4.10.3 1.9.8 - onerror - true - onerror - - - INFO - - - Lucene Snapshots - https://download.elasticsearch.org/lucenesnapshots/1637985/ - - - sonatype - http://oss.sonatype.org/content/repositories/releases/ - - - org.hamcrest - hamcrest-core - 1.3.RC2 - test - - - org.hamcrest - hamcrest-library - 1.3.RC2 - test + hamcrest-all com.carrotsearch.randomizedtesting randomizedtesting-runner - 2.1.10 - test org.apache.lucene lucene-test-framework - ${lucene.maven.version} - test - org.elasticsearch elasticsearch - ${elasticsearch.version} provided + org.apache.lucene lucene-core @@ -94,17 +61,16 @@ provided + com.amazonaws aws-java-sdk-ec2 ${amazonaws.version} - compile com.amazonaws aws-java-sdk-s3 ${amazonaws.version} - compile @@ -118,19 +84,14 @@ log4j log4j - 1.2.17 test - true org.elasticsearch elasticsearch - ${elasticsearch.version} test-jar - test - @@ -144,16 +105,10 @@ org.apache.maven.plugins maven-compiler-plugin - 3.0 - - 1.7 - 1.7 - com.carrotsearch.randomizedtesting junit4-maven-plugin - 2.0.14 tests @@ -241,27 +196,14 @@ org.apache.maven.plugins maven-surefire-plugin - 2.13 - - true - org.apache.maven.plugins maven-source-plugin - 2.2.1 - - - attach-sources - - jar - - - + org.apache.maven.plugins maven-assembly-plugin - 2.4 false ${project.build.directory}/releases/ From d056dec7742299d8051668eff4ccfc2a9e289ecc Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 2 Jan 2015 21:10:01 +0100 Subject: [PATCH 56/86] Add sonatype snapshot repository (cherry picked from commit d36e24a) --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index 12c60d0f..7a4051fc 100644 --- a/pom.xml +++ b/pom.xml @@ -222,4 +222,12 @@ + + + + oss-snapshots + Sonatype OSS Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + From 910001c480f5f8e49d25cc23639f3a8fcf7a8185 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 2 Jan 2015 22:28:50 +0100 Subject: [PATCH 57/86] Add China region for s3 and ec2 China region name is `cn-north-1`. Closes #156. (cherry picked from commit ea9609c) --- README.md | 4 ++-- src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java | 2 ++ .../org/elasticsearch/cloud/aws/InternalAwsS3Service.java | 4 ++++ .../java/org/elasticsearch/repositories/s3/S3Repository.java | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b203f7f..e7281d28 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ The `cloud.aws.region` can be set to a region and will automatically use the rel * `ap-northeast` (`ap-northeast-1`) * `eu-west` (`eu-west-1`) * `eu-central` (`eu-central-1`) -* `sa-east` (`sa-east-1`). - +* `sa-east` (`sa-east-1`) +* `cn-north` (`cn-north-1`) ## EC2 Discovery diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index dd15e608..8174e169 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -125,6 +125,8 @@ public synchronized AmazonEC2 client() { endpoint = "ec2.eu-central-1.amazonaws.com"; } else if (region.equals("sa-east") || region.equals("sa-east-1")) { endpoint = "ec2.sa-east-1.amazonaws.com"; + } else if (region.equals("cn-north") || region.equals("cn-north-1")) { + endpoint = "ec2.cn-north-1.amazonaws.com"; } else { throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index e436c107..4c17c921 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -191,6 +191,10 @@ private static String getEndpoint(String region) { return "s3-sa-east-1.amazonaws.com"; } else if ("sa-east-1".equals(region)) { return "s3-sa-east-1.amazonaws.com"; + } else if ("cn-north".equals(region)) { + return "s3-cn-north-1.amazonaws.com"; + } else if ("cn-north-1".equals(region)) { + return "s3-cn-north-1.amazonaws.com"; } else { throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); } diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 6b19a550..f93f3039 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -121,6 +121,10 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, region = "sa-east-1"; } else if ("sa-east-1".equals(regionSetting)) { region = "sa-east-1"; + } else if ("cn-north".equals(regionSetting)) { + region = "cn-north-1"; + } else if ("cn-north-1".equals(regionSetting)) { + region = "cn-north-1"; } } } From 630f0af0d15d3d9d89fea2f9fb3c219bcfed70e2 Mon Sep 17 00:00:00 2001 From: Serhiy Suprun Date: Wed, 18 Feb 2015 15:50:58 +0100 Subject: [PATCH 58/86] Fixed address resolution (cherry picked from commit e05036f) --- .../elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java index 1722bd77..ee8fa96d 100644 --- a/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java @@ -156,7 +156,7 @@ public List buildDynamicNodes() { address = instance.getPublicDnsName(); break; case PUBLIC_IP: - address = instance.getPublicDnsName(); + address = instance.getPublicIpAddress(); break; } if (address != null) { From 2f5e1af5925c4f3168b126da4957c1326665017d Mon Sep 17 00:00:00 2001 From: Andreas Kohn Date: Tue, 10 Feb 2015 16:50:00 +0100 Subject: [PATCH 59/86] Remove an excess quote character in an exception message (cherry picked from commit c7187f2) --- .../java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index fcb9e67f..49d947e9 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -67,7 +67,7 @@ public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable this.bufferSize = (bufferSize != null) ? bufferSize : MIN_BUFFER_SIZE; if (this.bufferSize.getBytes() < MIN_BUFFER_SIZE.getBytes()) { - throw new BlobStoreException("\"Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]"); + throw new BlobStoreException("Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]"); } this.numberOfRetries = maxRetries; From f12cacc7c1837993a2fccd1a1044815a7b2cf370 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 19 Feb 2015 16:50:06 +0100 Subject: [PATCH 60/86] Wrong endpoint for `cn-north-1` Commit ea9609c642c84309b6df08bc6103cf227719c2ab does not work as expected. China is not part of the [S3 regions](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) but part of [Isolated Regions](http://docs.aws.amazon.com/general/latest/gr/isolated_regions.html). The patch uses for S3 `s3-cn-north-1.amazonaws.com` which is wrong because in that case the end point is `s3.cn-north-1.amazonaws.com.cn`. Same goes for EC2 which was `ec2.cn-north-1.amazonaws.com` but should be `ec2.cn-north-1.amazonaws.com.cn`. Related documentation: * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region * http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region * http://docs.aws.amazon.com/general/latest/gr/isolated_regions.html Closing #156 (cherry picked from commit 491a59b) --- .../cloud/aws/AwsEc2Service.java | 2 +- .../cloud/aws/InternalAwsS3Service.java | 34 +++++-------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index 8174e169..963bd3e9 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -126,7 +126,7 @@ public synchronized AmazonEC2 client() { } else if (region.equals("sa-east") || region.equals("sa-east-1")) { endpoint = "ec2.sa-east-1.amazonaws.com"; } else if (region.equals("cn-north") || region.equals("cn-north-1")) { - endpoint = "ec2.cn-north-1.amazonaws.com"; + endpoint = "ec2.cn-north-1.amazonaws.com.cn"; } else { throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 4c17c921..4b7e45cf 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -159,42 +159,26 @@ private String getDefaultEndpoint() { } private static String getEndpoint(String region) { - if ("us-east".equals(region)) { + if ("us-east".equals(region) || "us-east-1".equals(region)) { return "s3.amazonaws.com"; - } else if ("us-east-1".equals(region)) { - return "s3.amazonaws.com"; - } else if ("us-west".equals(region)) { - return "s3-us-west-1.amazonaws.com"; - } else if ("us-west-1".equals(region)) { + } else if ("us-west".equals(region) || "us-west-1".equals(region)) { return "s3-us-west-1.amazonaws.com"; } else if ("us-west-2".equals(region)) { return "s3-us-west-2.amazonaws.com"; - } else if ("ap-southeast".equals(region)) { - return "s3-ap-southeast-1.amazonaws.com"; - } else if ("ap-southeast-1".equals(region)) { + } else if ("ap-southeast".equals(region) || "ap-southeast-1".equals(region)) { return "s3-ap-southeast-1.amazonaws.com"; } else if ("ap-southeast-2".equals(region)) { return "s3-ap-southeast-2.amazonaws.com"; - } else if ("ap-northeast".equals(region)) { - return "s3-ap-northeast-1.amazonaws.com"; - } else if ("ap-northeast-1".equals(region)) { + } else if ("ap-northeast".equals(region) || "ap-northeast-1".equals(region)) { return "s3-ap-northeast-1.amazonaws.com"; - } else if ("eu-west".equals(region)) { + } else if ("eu-west".equals(region) || "eu-west-1".equals(region)) { return "s3-eu-west-1.amazonaws.com"; - } else if ("eu-west-1".equals(region)) { - return "s3-eu-west-1.amazonaws.com"; - } else if ("eu-central".equals(region)) { - return "s3.eu-central-1.amazonaws.com"; - } else if ("eu-central-1".equals(region)) { + } else if ("eu-central".equals(region) || "eu-central-1".equals(region)) { return "s3.eu-central-1.amazonaws.com"; - } else if ("sa-east".equals(region)) { - return "s3-sa-east-1.amazonaws.com"; - } else if ("sa-east-1".equals(region)) { + } else if ("sa-east".equals(region) || "sa-east-1".equals(region)) { return "s3-sa-east-1.amazonaws.com"; - } else if ("cn-north".equals(region)) { - return "s3-cn-north-1.amazonaws.com"; - } else if ("cn-north-1".equals(region)) { - return "s3-cn-north-1.amazonaws.com"; + } else if ("cn-north".equals(region) || "cn-north-1".equals(region)) { + return "s3.cn-north-1.amazonaws.com.cn"; } else { throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); } From 5aa20a48b32a3f0ddc2ed502480c22f18c1a44bb Mon Sep 17 00:00:00 2001 From: Marcin Matlaszek Date: Thu, 5 Feb 2015 11:49:55 +0100 Subject: [PATCH 61/86] Fix region settings for s3 snapshot repository in eu-west-1 Closes #169. (cherry picked from commit 68fea58) --- .../java/org/elasticsearch/repositories/s3/S3Repository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index f93f3039..79f6b958 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -110,9 +110,9 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, } else if ("ap-northeast-1".equals(regionSetting)) { region = "ap-northeast-1"; } else if ("eu-west".equals(regionSetting)) { - region = "EU"; + region = "eu-west-1"; } else if ("eu-west-1".equals(regionSetting)) { - region = "EU"; + region = "eu-west-1"; } else if ("eu-central".equals(regionSetting)) { region = "eu-central-1"; } else if ("eu-central-1".equals(regionSetting)) { From 2c67e6496b94a81fa9ebd00214d32b2e100bfd55 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 19 Feb 2015 21:23:59 +0100 Subject: [PATCH 62/86] Simplify region setting code (cherry picked from commit de2ab73) --- .../repositories/s3/S3Repository.java | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 79f6b958..ac097683 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -88,42 +88,26 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, String regionSetting = repositorySettings.settings().get("cloud.aws.region", settings.get("cloud.aws.region")); if (regionSetting != null) { regionSetting = regionSetting.toLowerCase(Locale.ENGLISH); - if ("us-east".equals(regionSetting)) { + if ("us-east".equals(regionSetting) || "us-east-1".equals(regionSetting)) { // Default bucket - setting region to null region = null; - } else if ("us-east-1".equals(regionSetting)) { - region = null; - } else if ("us-west".equals(regionSetting)) { - region = "us-west-1"; - } else if ("us-west-1".equals(regionSetting)) { + } else if ("us-west".equals(regionSetting) || "us-west-1".equals(regionSetting)) { region = "us-west-1"; } else if ("us-west-2".equals(regionSetting)) { region = "us-west-2"; - } else if ("ap-southeast".equals(regionSetting)) { - region = "ap-southeast-1"; - } else if ("ap-southeast-1".equals(regionSetting)) { + } else if ("ap-southeast".equals(regionSetting) || "ap-southeast-1".equals(regionSetting)) { region = "ap-southeast-1"; } else if ("ap-southeast-2".equals(regionSetting)) { region = "ap-southeast-2"; - } else if ("ap-northeast".equals(regionSetting)) { - region = "ap-northeast-1"; - } else if ("ap-northeast-1".equals(regionSetting)) { + } else if ("ap-northeast".equals(regionSetting) || "ap-northeast-1".equals(regionSetting)) { region = "ap-northeast-1"; - } else if ("eu-west".equals(regionSetting)) { + } else if ("eu-west".equals(regionSetting) || "eu-west-1".equals(regionSetting)) { region = "eu-west-1"; - } else if ("eu-west-1".equals(regionSetting)) { - region = "eu-west-1"; - } else if ("eu-central".equals(regionSetting)) { - region = "eu-central-1"; - } else if ("eu-central-1".equals(regionSetting)) { + } else if ("eu-central".equals(regionSetting) || "eu-central-1".equals(regionSetting)) { region = "eu-central-1"; - } else if ("sa-east".equals(regionSetting)) { + } else if ("sa-east".equals(regionSetting) || "sa-east-1".equals(regionSetting)) { region = "sa-east-1"; - } else if ("sa-east-1".equals(regionSetting)) { - region = "sa-east-1"; - } else if ("cn-north".equals(regionSetting)) { - region = "cn-north-1"; - } else if ("cn-north-1".equals(regionSetting)) { + } else if ("cn-north".equals(regionSetting) || "cn-north-1".equals(regionSetting)) { region = "cn-north-1"; } } From ff6b23bd0f5d973a9d48651d99c8c9f9e2734ef4 Mon Sep 17 00:00:00 2001 From: Jerry Date: Wed, 4 Feb 2015 10:42:49 +0100 Subject: [PATCH 63/86] [doc] secret_key: should be Closes #168. (cherry picked from commit 3a4e472) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e7281d28..11fe5194 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ repositories: private-bucket: bucket: access_key: - secret_key: + secret_key: remote-bucket: bucket: region: From 70e694ba7e8c372a806db972447c3b9a38519fdd Mon Sep 17 00:00:00 2001 From: paul-e-cooley Date: Fri, 20 Feb 2015 22:08:43 -0800 Subject: [PATCH 64/86] Separate proxy traffic for ec2 and s3 Based on PR #178 by @paul-e-cooley. Thanks Paul! In addition to: ```yaml cloud: aws: protocol: https proxy_host: proxy1.company.com proxy_port: 8083 ``` You can also set different proxies for `ec2` and `s3`: ```yaml cloud: aws: s3: proxy_host: proxy1.company.com proxy_port: 8083 ec2: proxy_host: proxy2.company.com proxy_port: 8083 ``` PR rebased on master and lastest changes about component settings removal. Documentation added. Changes in tests. If a proxy is provided we run the tests, otherwise we ignore the test. Closes #177. (cherry picked from commit cb7beec) --- README.md | 12 +++++ .../cloud/aws/AwsEc2Service.java | 6 ++- .../cloud/aws/InternalAwsS3Service.java | 6 ++- ...S3ProxiedSnapshotRestoreOverHttpsTest.java | 47 +++++++++++++++++++ 4 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/elasticsearch/repositories/s3/S3ProxiedSnapshotRestoreOverHttpsTest.java diff --git a/README.md b/README.md index 11fe5194..0992b242 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,18 @@ cloud: proxy_port: 8083 ``` +You can also set different proxies for `ec2` and `s3`: + +``` +cloud: + aws: + s3: + proxy_host: proxy1.company.com + proxy_port: 8083 + ec2: + proxy_host: proxy2.company.com + proxy_port: 8083 +``` ### Region diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index 963bd3e9..02c58fe9 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -72,9 +72,11 @@ public synchronized AmazonEC2 client() { String account = componentSettings.get("access_key", settings.get("cloud.account")); String key = componentSettings.get("secret_key", settings.get("cloud.key")); - String proxyHost = componentSettings.get("proxy_host"); + String proxyHost = settings.get("cloud.aws.proxy_host"); + proxyHost = settings.get("cloud.aws.ec2.proxy_host", proxyHost); if (proxyHost != null) { - String portString = componentSettings.get("proxy_port", "80"); + String portString = settings.get("cloud.aws.proxy_port", "80"); + portString = settings.get("cloud.aws.ec2.proxy_port", portString); Integer proxyPort; try { proxyPort = Integer.parseInt(portString, 10); diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 4b7e45cf..21ac4db8 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -106,9 +106,11 @@ private synchronized AmazonS3 getClient(String endpoint, String protocol, String throw new ElasticsearchIllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]"); } - String proxyHost = componentSettings.get("proxy_host"); + String proxyHost = settings.get("cloud.aws.proxy_host"); + proxyHost = settings.get("cloud.aws.s3.proxy_host", proxyHost); if (proxyHost != null) { - String portString = componentSettings.get("proxy_port", "80"); + String portString = settings.get("cloud.aws.proxy_port", "80"); + portString = settings.get("cloud.aws.s3.proxy_port", portString); Integer proxyPort; try { proxyPort = Integer.parseInt(portString, 10); diff --git a/src/test/java/org/elasticsearch/repositories/s3/S3ProxiedSnapshotRestoreOverHttpsTest.java b/src/test/java/org/elasticsearch/repositories/s3/S3ProxiedSnapshotRestoreOverHttpsTest.java new file mode 100644 index 00000000..80b6e123 --- /dev/null +++ b/src/test/java/org/elasticsearch/repositories/s3/S3ProxiedSnapshotRestoreOverHttpsTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.s3; + +import org.elasticsearch.common.settings.Settings; +import org.junit.Before; + +/** + * This will only run if you define in your `elasticsearch.yml` file a s3 specific proxy + * cloud.aws.s3.proxy_host: mys3proxy.company.com + * cloud.aws.s3.proxy_port: 8080 + */ +public class S3ProxiedSnapshotRestoreOverHttpsTest extends AbstractS3SnapshotRestoreTest { + + private boolean proxySet = false; + + @Override + public Settings nodeSettings(int nodeOrdinal) { + Settings settings = super.nodeSettings(nodeOrdinal); + String proxyHost = settings.get("cloud.aws.s3.proxy_host"); + proxySet = proxyHost != null; + return settings; + } + + @Before + public void checkProxySettings() { + assumeTrue("we are expecting proxy settings in elasticsearch.yml file", proxySet); + } + +} From cb5148448f2bb45d00236b453e2179cf1abd60b1 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 16 Mar 2015 12:28:19 -0700 Subject: [PATCH 65/86] Create `es-1.5` branch --- README.md | 2 +- pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0992b242..1aebbd4e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.5.0-SNAPSHOT for Elasticsearch: 1.x +## Version 2.6.0-SNAPSHOT for Elasticsearch: 1.x If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index 7a4051fc..ad206a8b 100644 --- a/pom.xml +++ b/pom.xml @@ -7,12 +7,12 @@ org.elasticsearch elasticsearch-parent - 1.5.0-SNAPSHOT + 1.6.0-SNAPSHOT org.elasticsearch elasticsearch-cloud-aws - 2.5.0-SNAPSHOT + 2.6.0-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 0f7430eacdaabd3a17030cfeab326597f01b5fbb Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 25 Mar 2015 09:55:36 +0100 Subject: [PATCH 66/86] Move parent after artifact coordinates --- pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index ad206a8b..bbf0721f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,12 +4,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - org.elasticsearch - elasticsearch-parent - 1.6.0-SNAPSHOT - - org.elasticsearch elasticsearch-cloud-aws 2.6.0-SNAPSHOT @@ -31,6 +25,12 @@ http://github.com/elasticsearch/elasticsearch-cloud-aws + + org.elasticsearch + elasticsearch-parent + 1.6.0-SNAPSHOT + + 1.9.8 From 2a4642ed9e7b7cd050bf0661cc40c0058c1cc231 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 25 Mar 2015 09:54:09 +0100 Subject: [PATCH 67/86] Move to elastic owner (cherry picked from commit ec9d00f) --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index bbf0721f..d20cacb1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. - https://github.com/elasticsearch/elasticsearch-cloud-aws/ + https://github.com/elastic/elasticsearch-cloud-aws/ 2009 @@ -20,9 +20,9 @@ - scm:git:git@github.com:elasticsearch/elasticsearch-cloud-aws.git - scm:git:git@github.com:elasticsearch/elasticsearch-cloud-aws.git - http://github.com/elasticsearch/elasticsearch-cloud-aws + scm:git:git@github.com:elastic/elasticsearch-cloud-aws.git + scm:git:git@github.com:elastic/elasticsearch-cloud-aws.git + http://github.com/elastic/elasticsearch-cloud-aws From 471f7553df030acc6667fd4ec5d46a331f817f08 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 9 Mar 2015 19:16:26 +0100 Subject: [PATCH 68/86] Update aws-sdk-java to 1.9.23 Release notes: * [1.9.23](https://aws.amazon.com/releasenotes/Java/0061646886197506) * [1.9.22](https://aws.amazon.com/releasenotes/Java/9983269688843537) * [1.9.21](https://aws.amazon.com/releasenotes/Java/0068314282259743) * [1.9.20.1](https://aws.amazon.com/releasenotes/Java/1468424988909152) * [1.9.20](https://aws.amazon.com/releasenotes/Java/4095479451567110) * [1.9.19](https://aws.amazon.com/releasenotes/Java/7152067748916749) * [1.9.18](https://aws.amazon.com/releasenotes/Java/8255923333358616) * [1.9.17](https://aws.amazon.com/releasenotes/Java/7976814583460560) * [1.9.16](https://aws.amazon.com/releasenotes/Java/7930704610010512) * [1.9.15](https://aws.amazon.com/releasenotes/Java/7196148869312092) * [1.9.14](https://aws.amazon.com/releasenotes/Java/5550164873229724) * [1.9.13](https://aws.amazon.com/releasenotes/Java/4588838060473154) * [1.9.12](https://aws.amazon.com/releasenotes/Java/4184046784243318) * [1.9.11](https://aws.amazon.com/releasenotes/Java/4181470878749498) * [1.9.10](https://aws.amazon.com/releasenotes/Java/7263787800932935) * [1.9.9](https://aws.amazon.com/releasenotes/Java/1369906126177804) (cherry picked from commit 16a2f39) (cherry picked from commit 5e13c68) --- pom.xml | 2 +- .../java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d20cacb1..2b02342b 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ - 1.9.8 + 1.9.23 diff --git a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java index 950aa72a..33b14fef 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -192,6 +192,11 @@ public void setObjectAcl(String bucketName, String key, String versionId, Canned delegate.setObjectAcl(bucketName, key, versionId, acl); } + @Override + public void setObjectAcl(SetObjectAclRequest setObjectAclRequest) throws AmazonClientException, AmazonServiceException { + delegate.setObjectAcl(setObjectAclRequest); + } + @Override public AccessControlList getBucketAcl(String bucketName) throws AmazonClientException, AmazonServiceException { return delegate.getBucketAcl(bucketName); From 9809af5b8a4e13df5cc6237523472e76e84da073 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 21 Apr 2015 10:26:45 +0200 Subject: [PATCH 69/86] Add EC2/S3 Signer API setting If you are using a compatible EC2 or S3 service, they might be using an older API to sign the requests. You can set your compatible signer API using `cloud.aws.signer` (or `cloud.aws.ec2.signer` and `cloud.aws.s3.signer`) with the right signer to use. Defaults to `AWS4SignerType`. Supported today (time when this commit is done): * `QueryStringSignerType` * `AWS3SignerType` * `AWS4SignerType` * `NoOpSignerType` Closes #155. (cherry picked from commit 33b18b4) --- README.md | 8 +++ .../cloud/aws/AwsEc2Service.java | 11 ++++ .../elasticsearch/cloud/aws/AwsSigner.java | 54 +++++++++++++++++++ .../cloud/aws/InternalAwsS3Service.java | 11 ++++ .../cloud/aws/AWSSignersTest.java | 54 +++++++++++++++++++ 5 files changed, 138 insertions(+) create mode 100644 src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java create mode 100644 src/test/java/org/elasticsearch/cloud/aws/AWSSignersTest.java diff --git a/README.md b/README.md index 1aebbd4e..12b9d9d8 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,14 @@ The `cloud.aws.region` can be set to a region and will automatically use the rel * `sa-east` (`sa-east-1`) * `cn-north` (`cn-north-1`) + +### EC2/S3 Signer API + +If you are using a compatible EC2 or S3 service, they might be using an older API to sign the requests. +You can set your compatible signer API using `cloud.aws.signer` (or `cloud.aws.ec2.signer` and `cloud.aws.s3.signer`) +with the right signer to use. Defaults to `AWS4SignerType`. + + ## EC2 Discovery ec2 discovery allows to use the ec2 APIs to perform automatic discovery (similar to multicast in non hostile multicast environments). Here is a simple sample configuration: diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index 02c58fe9..2d9989af 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -86,6 +86,17 @@ public synchronized AmazonEC2 client() { clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); } + // #155: we might have 3rd party users using older EC2 API version + String awsSigner = settings.get("cloud.aws.ec2.signer", settings.get("cloud.aws.signer")); + if (awsSigner != null) { + logger.debug("using AWS API signer [{}]", awsSigner); + try { + AwsSigner.configureSigner(awsSigner, clientConfiguration); + } catch (ElasticsearchIllegalArgumentException e) { + logger.warn("wrong signer set for [cloud.aws.ec2.signer] or [cloud.aws.signer]: [{}]", awsSigner); + } + } + AWSCredentialsProvider credentials; if (account == null && key == null) { diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java b/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java new file mode 100644 index 00000000..c05e81ee --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.SignerFactory; +import org.elasticsearch.ElasticsearchIllegalArgumentException; + +public class AwsSigner { + + private AwsSigner() { + + } + + /** + * Add a AWS API Signer. + * @param signer Signer to use + * @param configuration AWS Client configuration + * @throws ElasticsearchIllegalArgumentException if signer does not exist + */ + public static void configureSigner(String signer, ClientConfiguration configuration) + throws ElasticsearchIllegalArgumentException { + + if (signer == null) { + throw new ElasticsearchIllegalArgumentException("[null] signer set"); + } + + try { + // We check this signer actually exists in AWS SDK + // It throws a IllegalArgumentException if not found + SignerFactory.getSignerByTypeAndService(signer, null); + configuration.setSignerOverride(signer); + } catch (IllegalArgumentException e) { + throw new ElasticsearchIllegalArgumentException("wrong signer set [" + signer + "]"); + } + } +} diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 21ac4db8..7e34b255 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -125,6 +125,17 @@ private synchronized AmazonS3 getClient(String endpoint, String protocol, String clientConfiguration.setMaxErrorRetry(maxRetries); } + // #155: we might have 3rd party users using older S3 API version + String awsSigner = settings.get("cloud.aws.s3.signer", settings.get("cloud.aws.signer")); + if (awsSigner != null) { + logger.debug("using AWS API signer [{}]", awsSigner); + try { + AwsSigner.configureSigner(awsSigner, clientConfiguration); + } catch (ElasticsearchIllegalArgumentException e) { + logger.warn("wrong signer set for [cloud.aws.s3.signer] or [cloud.aws.signer]: [{}]", awsSigner); + } + } + AWSCredentialsProvider credentials; if (account == null && key == null) { diff --git a/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTest.java b/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTest.java new file mode 100644 index 00000000..7aa08711 --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws; + +import com.amazonaws.ClientConfiguration; +import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; + +public class AWSSignersTest extends ElasticsearchTestCase { + + @Test + public void testSigners() { + assertThat(signerTester(null), is(false)); + assertThat(signerTester("QueryStringSignerType"), is(true)); + assertThat(signerTester("AWS3SignerType"), is(true)); + assertThat(signerTester("AWS4SignerType"), is(true)); + assertThat(signerTester("NoOpSignerType"), is(true)); + assertThat(signerTester("UndefinedSigner"), is(false)); + } + + /** + * Test a signer configuration + * @param signer signer name + * @return true if successful, false otherwise + */ + private boolean signerTester(String signer) { + try { + AwsSigner.configureSigner(signer, new ClientConfiguration()); + return true; + } catch (ElasticsearchIllegalArgumentException e) { + return false; + } + } +} From 7a8c7b0ed5601dc71b8d3376cd197b8e0db2bbbd Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 21 Apr 2015 21:22:09 +0200 Subject: [PATCH 70/86] Fix non working update dynamic settings Described in https://github.com/elastic/elasticsearch/issues/10614, it's not possible with cloud discovery plugin to update dynamic settings anymore. ```sh curl -XPUT localhost:9200/_cluster/settings -d '{ "persistent" : { "discovery.zen.minimum_master_nodes" : 3 }, "transient" : { "discovery.zen.minimum_master_nodes" : 3 } }' ``` gives ```json {"acknowledged":true,"persistent":{},"transient":{}} ``` This patch makes that working again. (cherry picked from commit 37d6897) --- .../discovery/ec2/Ec2Discovery.java | 3 +- .../ec2/Ec2DiscoveryUpdateSettingsITest.java | 63 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java diff --git a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index 2d44e1a2..098222c8 100755 --- a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.cluster.settings.ClusterDynamicSettings; import org.elasticsearch.cluster.settings.DynamicSettings; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; @@ -47,7 +48,7 @@ public class Ec2Discovery extends ZenDiscovery { public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, DiscoverySettings discoverySettings, - ElectMasterService electMasterService, DynamicSettings dynamicSettings, + ElectMasterService electMasterService, @ClusterDynamicSettings DynamicSettings dynamicSettings, Version version) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); diff --git a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java new file mode 100644 index 00000000..ca1b4d4e --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.ec2; + + +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.cloud.aws.AbstractAwsTest; +import org.elasticsearch.cloud.aws.AbstractAwsTest.AwsTest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; +import org.junit.Test; + +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.hamcrest.CoreMatchers.is; + +/** + * Just an empty Node Start test to check eveything if fine when + * starting. + * This test requires AWS to run. + */ +@AwsTest +@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) +public class Ec2DiscoveryUpdateSettingsITest extends AbstractAwsTest { + + @Test + public void testMinimumMasterNodesStart() { + Settings nodeSettings = settingsBuilder() + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("cloud.enabled", true) + .put("discovery.type", "ec2") + .build(); + internalCluster().startNode(nodeSettings); + + // We try to update minimum_master_nodes now + ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(settingsBuilder().put("discovery.zen.minimum_master_nodes", 1)) + .setTransientSettings(settingsBuilder().put("discovery.zen.minimum_master_nodes", 1)) + .get(); + + Integer min = response.getPersistentSettings().getAsInt("discovery.zen.minimum_master_nodes", null); + assertThat(min, is(1)); + } + +} From dc5947d69599e82dd513d63aff4aacb3826633d3 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 8 May 2015 12:32:26 -0400 Subject: [PATCH 71/86] Disable response metadata cache. This cache is only used for diagnostic purposes, but can force objects from every response to the old generation. Fixes #193 Conflicts: src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java --- pom.xml | 2 +- .../cloud/aws/AwsEc2Service.java | 3 +++ .../cloud/aws/AmazonS3Wrapper.java | 20 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b02342b..c611f5d3 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ - 1.9.23 + 1.9.34 diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index 2d9989af..0fa876a0 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -60,6 +60,9 @@ public synchronized AmazonEC2 client() { } ClientConfiguration clientConfiguration = new ClientConfiguration(); + // the response metadata cache is only there for diagnostics purposes, + // but can force objects from every response to the old generation. + clientConfiguration.setResponseMetadataCacheSize(0); String protocol = componentSettings.get("protocol", "https").toLowerCase(); protocol = componentSettings.get("ec2.protocol", protocol).toLowerCase(); if ("http".equals(protocol)) { diff --git a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java index 33b14fef..ce071bc6 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -257,6 +257,26 @@ public void deleteBucket(String bucketName) throws AmazonClientException, Amazon delegate.deleteBucket(bucketName); } + @Override + public void setBucketReplicationConfiguration(String bucketName, BucketReplicationConfiguration configuration) throws AmazonServiceException, AmazonClientException { + delegate.setBucketReplicationConfiguration(bucketName, configuration); + } + + @Override + public void setBucketReplicationConfiguration(SetBucketReplicationConfigurationRequest setBucketReplicationConfigurationRequest) throws AmazonServiceException, AmazonClientException { + delegate.setBucketReplicationConfiguration(setBucketReplicationConfigurationRequest); + } + + @Override + public BucketReplicationConfiguration getBucketReplicationConfiguration(String bucketName) throws AmazonServiceException, AmazonClientException { + return delegate.getBucketReplicationConfiguration(bucketName); + } + + @Override + public void deleteBucketReplicationConfiguration(String bucketName) throws AmazonServiceException, AmazonClientException { + delegate.deleteBucketReplicationConfiguration(bucketName); + } + @Override public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException { return delegate.putObject(putObjectRequest); From 169cde2c7c31086c02eba569c321ff7921b4b9b7 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 8 May 2015 12:41:43 -0400 Subject: [PATCH 72/86] Disable response metadata cache in this ClientConfiguration as well. --- .../java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 7e34b255..e2f10b23 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -93,6 +93,9 @@ private synchronized AmazonS3 getClient(String endpoint, String protocol, String } ClientConfiguration clientConfiguration = new ClientConfiguration(); + // the response metadata cache is only there for diagnostics purposes, + // but can force objects from every response to the old generation. + clientConfiguration.setResponseMetadataCacheSize(0); if (protocol == null) { protocol = componentSettings.get("protocol", "https").toLowerCase(); protocol = componentSettings.get("s3.protocol", protocol).toLowerCase(); From e72af1937f8e4079ecc983c79e2d426ad075f382 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 20 May 2015 16:58:35 +0200 Subject: [PATCH 73/86] [doc] correct S3 policy for multiparts for multipart to work correctly you need to also include the necessary actions in the policy. ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "s3:ListBucket", "s3:GetBucketLocation", "s3:ListBucketMultipartUploads", "s3:ListBucketVersions" ], "Resource": [ "arn:aws:s3:::yourbucket" ] }, { "Effect": "Allow", "Action": [ "s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:PutObjectAcl", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts" ], "Resource": [ "arn:aws:s3:::yourbucket/*" ] } ] } ``` Closes #214 (cherry picked from commit 62966ad) (cherry picked from commit 25a37a5) --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 12b9d9d8..b82debd1 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,10 @@ In order to restrict the Elasticsearch snapshot process to the minimum required "Statement": [ { "Action": [ - "s3:ListBucket" + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketMultipartUploads", + "s3:ListBucketVersions" ], "Effect": "Allow", "Resource": [ @@ -197,7 +200,9 @@ In order to restrict the Elasticsearch snapshot process to the minimum required "Action": [ "s3:GetObject", "s3:PutObject", - "s3:DeleteObject" + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" ], "Effect": "Allow", "Resource": [ @@ -207,7 +212,6 @@ In order to restrict the Elasticsearch snapshot process to the minimum required ], "Version": "2012-10-17" } - ``` You may further restrict the permissions by specifying a prefix within the bucket, in this example, named "foo". @@ -217,7 +221,10 @@ You may further restrict the permissions by specifying a prefix within the bucke "Statement": [ { "Action": [ - "s3:ListBucket" + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketMultipartUploads", + "s3:ListBucketVersions" ], "Condition": { "StringLike": { @@ -235,7 +242,9 @@ You may further restrict the permissions by specifying a prefix within the bucke "Action": [ "s3:GetObject", "s3:PutObject", - "s3:DeleteObject" + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" ], "Effect": "Allow", "Resource": [ @@ -245,7 +254,6 @@ You may further restrict the permissions by specifying a prefix within the bucke ], "Version": "2012-10-17" } - ``` The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository registration will fail. If you want elasticsearch to create the bucket instead, you can add the permission to create a specific bucket like this: From 3369e025cc79cc4eb8f43a2b26e9f6ee54d90f35 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 4 Mar 2015 16:50:25 +0100 Subject: [PATCH 74/86] Check MD5 while doing snapshot There is a feature available in S3 that clients can use to ensure data integrity on upload. Whenever an object is PUT to an S3 bucket, the client is able to get back the `MD5` base64 encoded and check that it's the same `MD5` as the local one. For reference, please see the [S3 PutObject API](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html). Closes #186. (cherry picked from commit 2d4fd39) --- .../aws/blobstore/DefaultS3OutputStream.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java index 04d30f2b..c26e8888 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java @@ -21,12 +21,18 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.model.*; +import com.amazonaws.util.Base64; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; @@ -49,7 +55,7 @@ public class DefaultS3OutputStream extends S3OutputStream { private static final ByteSizeValue MULTIPART_MAX_SIZE = new ByteSizeValue(5, ByteSizeUnit.GB); - + private static final ESLogger logger = Loggers.getLogger("cloud.aws"); /** * Multipart Upload API data */ @@ -120,7 +126,28 @@ protected void doUpload(S3BlobStore blobStore, String bucketName, String blobNam md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } md.setContentLength(length); - blobStore.client().putObject(bucketName, blobName, is, md); + + InputStream inputStream = is; + + // We try to compute a MD5 while reading it + MessageDigest messageDigest; + try { + messageDigest = MessageDigest.getInstance("MD5"); + inputStream = new DigestInputStream(is, messageDigest); + } catch (NoSuchAlgorithmException impossible) { + // Every implementation of the Java platform is required to support MD5 (see MessageDigest) + throw new RuntimeException(impossible); + } + PutObjectResult putObjectResult = blobStore.client().putObject(bucketName, blobName, inputStream, md); + + String localMd5 = Base64.encodeAsString(messageDigest.digest()); + String remoteMd5 = putObjectResult.getContentMd5(); + if (!localMd5.equals(remoteMd5)) { + logger.debug("MD5 local [{}], remote [{}] are not equal...", localMd5, remoteMd5); + throw new AmazonS3Exception("MD5 local [" + localMd5 + + "], remote [" + remoteMd5 + + "] are not equal..."); + } } private void initializeMultipart() { From 902dffca20b14fa776c88860358e6eeff66fe169 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 10 Jun 2015 16:51:20 +0200 Subject: [PATCH 75/86] Update to elasticsearch 1.7.0-SNAPSHOT --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index c611f5d3..bbd15e5d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.elasticsearch elasticsearch-cloud-aws - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. @@ -28,7 +28,7 @@ org.elasticsearch elasticsearch-parent - 1.6.0-SNAPSHOT + 1.7.0-SNAPSHOT From 1ebe6dd119d35eb07060158612cc6e636f330a6b Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 17 Jul 2015 16:09:50 +0200 Subject: [PATCH 76/86] Update to Elasticsearch 1.7.0 Closes #225 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bbd15e5d..28440e72 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.elasticsearch elasticsearch-parent - 1.7.0-SNAPSHOT + 1.7.0 From 5feb58ce55509817d96c7a210d1544cd4542be83 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 17 Jul 2015 16:22:34 +0200 Subject: [PATCH 77/86] Do not embed and serialize AmazonClientExceptions Only serializable exceptions are allowed to be transported over the wire in ES 1.7 --- .../aws/blobstore/DefaultS3OutputStream.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java index c26e8888..83171b68 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java @@ -112,7 +112,8 @@ private void upload(byte[] bytes, int off, int len) throws IOException { is.reset(); retry++; } else { - throw new IOException("Unable to upload object " + getBlobName(), e); + // Since ES 1.7, third party exceptions like AmazonClientException cannot be serialized and thus should not be embedded in IOException + throw new IOException("Unable to upload object " + getBlobName() + " due to " + e.getClass().getSimpleName() + ": " + e.getMessage()); } } } @@ -150,7 +151,7 @@ protected void doUpload(S3BlobStore blobStore, String bucketName, String blobNam } } - private void initializeMultipart() { + private void initializeMultipart() throws IOException { int retry = 0; while ((retry <= getNumberOfRetries()) && (multipartId == null)) { try { @@ -163,7 +164,9 @@ private void initializeMultipart() { if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { retry++; } else { - throw e; + // Since ES 1.7, third party exceptions like AmazonClientException cannot be serialized and thus should not be embedded in IOException + throw new IOException("Unable to initialize multipart request for object " + getBlobName() + + " due to " + e.getClass().getSimpleName() + ": " + e.getMessage()); } } } @@ -194,7 +197,9 @@ private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) t retry++; } else { abortMultipart(); - throw e; + // Since ES 1.7, third party exceptions like AmazonClientException cannot be serialized and thus should not be embedded in IOException + throw new IOException("Unable to upload multipart request [" + multipartId + "] for object " + getBlobName() + + " due to " + e.getClass().getSimpleName() + ": " + e.getMessage()); } } } @@ -217,7 +222,7 @@ protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, S } - private void completeMultipart() { + private void completeMultipart() throws IOException { int retry = 0; while (retry <= getNumberOfRetries()) { try { @@ -229,7 +234,9 @@ private void completeMultipart() { retry++; } else { abortMultipart(); - throw e; + // Since ES 1.7, third party exceptions like AmazonClientException cannot be serialized and thus should not be embedded in IOException + throw new IOException("Unable to complete multipart request [" + multipartId + "] for object " + getBlobName() + + " due to " + e.getClass().getSimpleName() + ": " + e.getMessage()); } } } From 6967ecc49cd2aa6a1b4be8b2f0dfea97c55f2026 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 17 Jul 2015 17:29:45 +0200 Subject: [PATCH 78/86] prepare release elasticsearch-cloud-aws-2.7.0 --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b82debd1..a482d8fe 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.6.0-SNAPSHOT for Elasticsearch: 1.x +## Version 2.7.0 for Elasticsearch: 1.7 If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index 28440e72..1d926cf0 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.elasticsearch elasticsearch-cloud-aws - 2.7.0-SNAPSHOT + 2.7.0 jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 2b55952efb7b881b665b02f581bf7f1a03dca470 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 17 Jul 2015 17:31:33 +0200 Subject: [PATCH 79/86] prepare for next development iteration --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a482d8fe..f341f85b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.7.0 for Elasticsearch: 1.7 +## Version 2.7.1-SNAPSHOT for Elasticsearch: 1.7 If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index 1d926cf0..13adc3f9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.elasticsearch elasticsearch-cloud-aws - 2.7.0 + 2.7.1-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 4e86bbf766accfc4e1af9580624a51e2e97cde5d Mon Sep 17 00:00:00 2001 From: Marc Melvin Date: Mon, 17 Aug 2015 23:06:32 +0200 Subject: [PATCH 80/86] Update aws-sdk-java to 1.10.11 See #234 (cherry picked from commit 0a8fb47) --- pom.xml | 2 +- .../cloud/aws/AmazonS3Wrapper.java | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 13adc3f9..c6b1adf7 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ - 1.9.34 + 1.10.11 diff --git a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java index ce071bc6..1c3a5ea7 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -576,4 +576,78 @@ public void disableRequesterPays(String bucketName) throws AmazonServiceExceptio public boolean isRequesterPaysEnabled(String bucketName) throws AmazonServiceException, AmazonClientException { return delegate.isRequesterPaysEnabled(bucketName); } + + + @Override + public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest) + throws AmazonClientException, AmazonServiceException { + return delegate.listNextBatchOfObjects(listNextBatchOfObjectsRequest); + } + + + @Override + public VersionListing listNextBatchOfVersions(ListNextBatchOfVersionsRequest listNextBatchOfVersionsRequest) + throws AmazonClientException, AmazonServiceException { + return delegate.listNextBatchOfVersions(listNextBatchOfVersionsRequest); + } + + + @Override + public Owner getS3AccountOwner(GetS3AccountOwnerRequest getS3AccountOwnerRequest) + throws AmazonClientException, AmazonServiceException { + return delegate.getS3AccountOwner(getS3AccountOwnerRequest); + } + + + @Override + public BucketLoggingConfiguration getBucketLoggingConfiguration( + GetBucketLoggingConfigurationRequest getBucketLoggingConfigurationRequest) + throws AmazonClientException, AmazonServiceException { + return delegate.getBucketLoggingConfiguration(getBucketLoggingConfigurationRequest); + } + + + @Override + public BucketVersioningConfiguration getBucketVersioningConfiguration( + GetBucketVersioningConfigurationRequest getBucketVersioningConfigurationRequest) + throws AmazonClientException, AmazonServiceException { + return delegate.getBucketVersioningConfiguration(getBucketVersioningConfigurationRequest); + } + + + @Override + public BucketLifecycleConfiguration getBucketLifecycleConfiguration( + GetBucketLifecycleConfigurationRequest getBucketLifecycleConfigurationRequest) { + return delegate.getBucketLifecycleConfiguration(getBucketLifecycleConfigurationRequest); + } + + + @Override + public BucketCrossOriginConfiguration getBucketCrossOriginConfiguration( + GetBucketCrossOriginConfigurationRequest getBucketCrossOriginConfigurationRequest) { + return delegate.getBucketCrossOriginConfiguration(getBucketCrossOriginConfigurationRequest); + } + + + @Override + public BucketTaggingConfiguration getBucketTaggingConfiguration( + GetBucketTaggingConfigurationRequest getBucketTaggingConfigurationRequest) { + return delegate.getBucketTaggingConfiguration(getBucketTaggingConfigurationRequest); + } + + + @Override + public BucketNotificationConfiguration getBucketNotificationConfiguration( + GetBucketNotificationConfigurationRequest getBucketNotificationConfigurationRequest) + throws AmazonClientException, AmazonServiceException { + return delegate.getBucketNotificationConfiguration(getBucketNotificationConfigurationRequest); + } + + + @Override + public BucketReplicationConfiguration getBucketReplicationConfiguration( + GetBucketReplicationConfigurationRequest getBucketReplicationConfigurationRequest) + throws AmazonServiceException, AmazonClientException { + return delegate.getBucketReplicationConfiguration(getBucketReplicationConfigurationRequest); + } } From a6d7787041fce918dbd7aeb02f451a419849c5e4 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 24 Aug 2015 14:18:35 +0200 Subject: [PATCH 81/86] Update aws-sdk-java to 1.10.12 Follow up for #234. Release notes: http://aws.amazon.com/releasenotes/Java/6635368276326731 Closes #236. (cherry picked from commit 51d70f2) (cherry picked from commit bc56248) (cherry picked from commit 9781068) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c6b1adf7..da1f8879 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ - 1.10.11 + 1.10.12 From 0db449ae35e3515e2301ec0fc34d9aa19a6d368b Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Aug 2015 21:39:53 +0200 Subject: [PATCH 82/86] Force inclusion of Joda 2.8.2 Follow up for discussion #234 We need to pull in Joda 2.8.2 and don't rely of the version declared by elasticsearch which is currently 2.7. It will fix issues seen with recent Java versions. See also https://github.com/aws/aws-sdk-java/issues/444 Closes #239. --- pom.xml | 15 +++++++++++++++ src/main/assemblies/plugin.xml | 1 + 2 files changed, 16 insertions(+) diff --git a/pom.xml b/pom.xml index da1f8879..bb86db62 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,12 @@ org.elasticsearch elasticsearch provided + + + joda-time + joda-time + + @@ -73,6 +79,15 @@ ${amazonaws.version} + + + + joda-time + joda-time + 2.8.2 + compile + + diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml index 532a360c..0e2f36c8 100644 --- a/src/main/assemblies/plugin.xml +++ b/src/main/assemblies/plugin.xml @@ -20,6 +20,7 @@ true com.amazonaws:aws-java-sdk + joda-time:joda-time commons-codec:commons-codec From 378f16de0e64f6e82d3578e8416572e7ee6a32f8 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 27 Aug 2015 16:05:00 +0200 Subject: [PATCH 83/86] prepare release elasticsearch-cloud-aws-2.7.1 --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f341f85b..f625c03c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.7.1-SNAPSHOT for Elasticsearch: 1.7 +## Version 2.7.1 for Elasticsearch: 1.7 If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index bb86db62..73d5fcbc 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.elasticsearch elasticsearch-cloud-aws - 2.7.1-SNAPSHOT + 2.7.1 jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 73b9252dba61f9216f6c423157c9fa3a0a472dd5 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 27 Aug 2015 16:06:36 +0200 Subject: [PATCH 84/86] prepare for next development iteration --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f625c03c..90eaad11 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AWS Cloud Plugin for Elasticsearch The Amazon Web Service (AWS) Cloud plugin allows to use [AWS API](https://github.com/aws/aws-sdk-java) for the unicast discovery mechanism and add S3 repositories. -## Version 2.7.1 for Elasticsearch: 1.7 +## Version 2.7.2-SNAPSHOT for Elasticsearch: 1.7 If you are looking for another version documentation, please refer to the [compatibility matrix](https://github.com/elasticsearch/elasticsearch-cloud-aws/#aws-cloud-plugin-for-elasticsearch). diff --git a/pom.xml b/pom.xml index 73d5fcbc..f0422c8e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.elasticsearch elasticsearch-cloud-aws - 2.7.1 + 2.7.2-SNAPSHOT jar Elasticsearch AWS cloud plugin The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. From 33366f968f24f24a05689e0abf951b6c23f78e08 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 28 Apr 2016 18:33:53 +0200 Subject: [PATCH 85/86] Update aws sdk to 1.10.69 and add cloud.aws.s3.throttle_retries setting This PR brings 2 changes: * Upgrade to AWS SDK 1.10.69 * Add cloud.aws.s3.throttle_retries setting * Moving from JSON.org to Jackson for request marshallers. * The Java SDK now supports retry throttling to limit the rate of retries during periods of reduced availability. This throttling behavior can be enabled via ClientConfiguration or via the system property "-Dcom.amazonaws.sdk.enableThrottledRetry". * Fixed String case conversion issues when running with non English locales. * AWS SDK for Java introduces a new dynamic endpoint system that can compute endpoints for services in new regions. * Introducing a new AWS region, ap-northeast-2. * Added a new metric, HttpSocketReadTime, that records socket read latency. You can enable this metric by adding enableHttpSocketReadMetric to the system property com.amazonaws.sdk.enableDefaultMetrics. For more information, see [Enabling Metrics with the AWS SDK for Java](https://java.awsblog.com/post/Tx3C0RV4NRRBKTG/Enabling-Metrics-with-the-AWS-SDK-for-Java). * New Client Execution timeout feature to set a limit spent across retries, backoffs, ummarshalling, etc. This new timeout can be specified at the client level or per request. Also included in this release is the ability to specify the existing HTTP Request timeout per request rather than just per client. * Added support for RequesterPays for all operations. * Ignore the 'Connection' header when generating S3 responses. * Allow users to generate an AmazonS3URI from a string without using URL encoding. * Fixed issue that prevented creating buckets when using a client configured for the s3-external-1 endpoint. * Amazon S3 bucket lifecycle configuration supports two new features: the removal of expired object delete markers and an action to abort incomplete multipart uploads. * Allow TransferManagerConfiguration to accept integer values for multipart upload threshold. * Copy the list of ETags before sorting aws/aws-sdk-java#589. * Option to disable chunked encoding aws/aws-sdk-java#586. * Adding retry on InternalErrors in CompleteMultipartUpload operation. aws/aws-sdk-java#538 * Deprecated two APIs : AmazonS3#changeObjectStorageClass and AmazonS3#setObjectRedirectLocation. * Added support for the aws-exec-read canned ACL. Owner gets FULL_CONTROL. Amazon EC2 gets READ access to GET an Amazon Machine Image (AMI) bundle from Amazon S3. * Added support for referencing security groups in peered Virtual Private Clouds (VPCs). For more information see the service announcement at https://aws.amazon.com/about-aws/whats-new/2016/03/announcing-support-for-security-group-references-in-a-peered-vpc/ . * Fixed a bug in AWS SDK for Java - Amazon EC2 module that returns NPE for dry run requests. * Regenerated client with new implementation of code generator. * This feature enables support for DNS resolution of public hostnames to private IP addresses when queried over ClassicLink. Additionally, you can now access private hosted zones associated with your VPC from a linked EC2-Classic instance. ClassicLink DNS support makes it easier for EC2-Classic instances to communicate with VPC resources using public DNS hostnames. * You can now use Network Address Translation (NAT) Gateway, a highly available AWS managed service that makes it easy to connect to the Internet from instances within a private subnet in an AWS Virtual Private Cloud (VPC). Previously, you needed to launch a NAT instance to enable NAT for instances in a private subnet. Amazon VPC NAT Gateway is available in the US East (N. Virginia), US West (Oregon), US West (N. California), EU (Ireland), Asia Pacific (Tokyo), Asia Pacific (Singapore), and Asia Pacific (Sydney) regions. To learn more about Amazon VPC NAT, see [New - Managed NAT (Network Address Translation) Gateway for AWS](https://aws.amazon.com/blogs/aws/new-managed-nat-network-address-translation-gateway-for-aws/) * A default read timeout is now applied when querying data from EC2 metadata service. Defaults to `true`. If anyone is having trouble with this option, you could disable it with `cloud.aws.s3.throttle_retries: false` in `elasticsearch.yml` file. Backport of elastic/elasticsearch#17784 for 1.7 series --- pom.xml | 2 +- .../cloud/aws/InternalAwsS3Service.java | 1 + .../cloud/aws/AmazonS3Wrapper.java | 21 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0422c8e..3775685c 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ - 1.10.12 + 1.10.69 diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index e2f10b23..fc9dbff4 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -126,6 +126,7 @@ private synchronized AmazonS3 getClient(String endpoint, String protocol, String if (maxRetries != null) { // If not explicitly set, default to 3 with exponential backoff policy clientConfiguration.setMaxErrorRetry(maxRetries); + clientConfiguration.setUseThrottleRetries(settings.getAsBoolean("cloud.aws.s3.throttle_retries", true)); } // #155: we might have 3rd party users using older S3 API version diff --git a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java index 1c3a5ea7..3909f9cb 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -122,6 +122,11 @@ public boolean doesBucketExist(String bucketName) throws AmazonClientException, return delegate.doesBucketExist(bucketName); } + @Override + public HeadBucketResult headBucket(HeadBucketRequest headBucketRequest) throws AmazonClientException, AmazonServiceException { + return delegate.headBucket(headBucketRequest); + } + @Override public List listBuckets() throws AmazonClientException, AmazonServiceException { return delegate.listBuckets(); @@ -172,6 +177,11 @@ public AccessControlList getObjectAcl(String bucketName, String key, String vers return delegate.getObjectAcl(bucketName, key, versionId); } + @Override + public AccessControlList getObjectAcl(GetObjectAclRequest getObjectAclRequest) throws AmazonClientException, AmazonServiceException { + return delegate.getObjectAcl(getObjectAclRequest); + } + @Override public void setObjectAcl(String bucketName, String key, AccessControlList acl) throws AmazonClientException, AmazonServiceException { delegate.setObjectAcl(bucketName, key, acl); @@ -277,6 +287,17 @@ public void deleteBucketReplicationConfiguration(String bucketName) throws Amazo delegate.deleteBucketReplicationConfiguration(bucketName); } + @Override + public void deleteBucketReplicationConfiguration(DeleteBucketReplicationConfigurationRequest request) throws AmazonServiceException, + AmazonClientException { + delegate.deleteBucketReplicationConfiguration(request); + } + + @Override + public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, AmazonClientException { + return delegate.doesObjectExist(bucketName, objectName); + } + @Override public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException { return delegate.putObject(putObjectRequest); From dd8535644cf4d5da94caad384d205ff079489d5b Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 27 May 2016 12:51:53 +0200 Subject: [PATCH 86/86] Change cloud.aws.s3.throttle_retries by repositories.s3.use_throttle_retries setting --- README.md | 5 ++++ .../elasticsearch/cloud/aws/AwsS3Service.java | 7 ++---- .../cloud/aws/InternalAwsS3Service.java | 24 +++++-------------- .../repositories/s3/S3Repository.java | 10 +++++--- .../cloud/aws/TestAwsS3Service.java | 16 +++---------- .../s3/AbstractS3SnapshotRestoreTest.java | 13 +++++----- 6 files changed, 30 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 90eaad11..b8448551 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,11 @@ The following settings are supported: * `server_side_encryption`: When set to `true` files are encrypted on server side using AES256 algorithm. Defaults to `false`. * `buffer_size`: Minimum threshold below which the chunk is uploaded using a single request. Beyond this threshold, the S3 repository will use the [AWS Multipart Upload API](http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html) to split the chunk into several parts, each of `buffer_size` length, and to upload each part in its own request. Note that positionning a buffer size lower than `5mb` is not allowed since it will prevents the use of the Multipart API and may result in upload errors. Defaults to `5mb`. * `max_retries`: Number of retries in case of S3 errors. Defaults to `3`. +* `use_throttle_retries`: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`). + +Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file. +They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories +by setting `repositories.s3.compress: true` in `elasticsearch.yml`. The S3 repositories are using the same credentials as the rest of the AWS services provided by this plugin (`discovery`). See [Generic Configuration](#generic-configuration) for details. diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index e5db2ed7..15fcef48 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -26,9 +26,6 @@ * */ public interface AwsS3Service extends LifecycleComponent { - AmazonS3 client(); - - AmazonS3 client(String endpoint, String protocol, String region, String account, String key); - - AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries); + AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, + boolean useThrottleRetries); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index fc9dbff4..f43bcf13 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -55,21 +55,8 @@ public InternalAwsS3Service(Settings settings, SettingsFilter settingsFilter) { } @Override - public synchronized AmazonS3 client() { - String endpoint = getDefaultEndpoint(); - String account = componentSettings.get("access_key", settings.get("cloud.account")); - String key = componentSettings.get("secret_key", settings.get("cloud.key")); - - return getClient(endpoint, null, account, key, null); - } - - @Override - public AmazonS3 client(String endpoint, String protocol, String region, String account, String key) { - return client(endpoint, protocol, region, account, key, null); - } - - @Override - public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) { + public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, + boolean useThrottleRetries) { if (region != null && endpoint == null) { endpoint = getEndpoint(region); logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); @@ -81,11 +68,12 @@ public synchronized AmazonS3 client(String endpoint, String protocol, String reg key = componentSettings.get("secret_key", settings.get("cloud.key")); } - return getClient(endpoint, protocol, account, key, maxRetries); + return getClient(endpoint, protocol, account, key, maxRetries, useThrottleRetries); } - private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries) { + private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries, + boolean useThrottleRetries) { Tuple clientDescriptor = new Tuple(endpoint, account); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { @@ -126,8 +114,8 @@ private synchronized AmazonS3 getClient(String endpoint, String protocol, String if (maxRetries != null) { // If not explicitly set, default to 3 with exponential backoff policy clientConfiguration.setMaxErrorRetry(maxRetries); - clientConfiguration.setUseThrottleRetries(settings.getAsBoolean("cloud.aws.s3.throttle_retries", true)); } + clientConfiguration.setUseThrottleRetries(useThrottleRetries); // #155: we might have 3rd party users using older S3 API version String awsSigner = settings.get("cloud.aws.s3.signer", settings.get("cloud.aws.signer")); diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index ac097683..92b34dcb 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -19,6 +19,7 @@ package org.elasticsearch.repositories.s3; +import com.amazonaws.ClientConfiguration; import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cloud.aws.blobstore.S3BlobStore; import org.elasticsearch.common.Strings; @@ -116,13 +117,16 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings, boolean serverSideEncryption = repositorySettings.settings().getAsBoolean("server_side_encryption", componentSettings.getAsBoolean("server_side_encryption", false)); ByteSizeValue bufferSize = repositorySettings.settings().getAsBytesSize("buffer_size", componentSettings.getAsBytesSize("buffer_size", null)); Integer maxRetries = repositorySettings.settings().getAsInt("max_retries", componentSettings.getAsInt("max_retries", 3)); + boolean useThrottleRetries = repositorySettings.settings().getAsBoolean("use_throttle_retries", settings.getAsBoolean("repositories.s3.use_throttle_retries", ClientConfiguration.DEFAULT_THROTTLE_RETRIES)); this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); - logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}]", - bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries); + logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " + + "buffer_size [{}], max_retries [{}], use_throttle_retries [{}]", + bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries); - blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key"), maxRetries), bucket, region, serverSideEncryption, bufferSize, maxRetries); + blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, repositorySettings.settings().get("access_key"), + repositorySettings.settings().get("secret_key"), maxRetries, useThrottleRetries), bucket, region, serverSideEncryption, bufferSize, maxRetries); String basePath = repositorySettings.settings().get("base_path", null); if (Strings.hasLength(basePath)) { BlobPath path = new BlobPath(); diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java index b8c9b2ee..256c20a9 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -38,20 +38,10 @@ public TestAwsS3Service(Settings settings, SettingsFilter settingsFilter) { super(settings, settingsFilter); } - - @Override - public synchronized AmazonS3 client() { - return cachedWrapper(super.client()); - } - - @Override - public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key) { - return cachedWrapper(super.client(endpoint, protocol, region, account, key)); - } - @Override - public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries) { - return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries)); + public synchronized AmazonS3 client(String endpoint, String protocol, String region, String account, String key, Integer maxRetries, + boolean useThrottleRetries) { + return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, useThrottleRetries)); } private AmazonS3 cachedWrapper(AmazonS3 client) { diff --git a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 4cab5316..1c6ae538 100644 --- a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -192,11 +192,12 @@ public void testEncryption() { Settings settings = internalCluster().getInstance(Settings.class); Settings bucket = settings.getByPrefix("repositories.s3."); AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( - null, - null, - bucket.get("region", settings.get("repositories.s3.region")), - bucket.get("access_key", settings.get("cloud.aws.access_key")), - bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); + null, + null, + bucket.get("region", settings.get("repositories.s3.region")), + bucket.get("access_key", settings.get("cloud.aws.access_key")), + bucket.get("secret_key", settings.get("cloud.aws.secret_key")), + null, randomBoolean()); String bucketName = bucket.get("bucket"); logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); @@ -460,7 +461,7 @@ public void cleanRepositoryFiles(String basePath) { // We check that settings has been set in elasticsearch.yml integration test file // as described in README assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); - AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey); + AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null, randomBoolean()); try { ObjectListing prevListing = null; //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html