From 5a7b8257a95713c12e32dd88d7c63f3e54201d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Hern=C3=A1ndez?= Date: Mon, 6 Jul 2020 00:34:15 +0200 Subject: [PATCH] [dynamodb] Improve endpoint and region config * Upgrade AWS SDK from version 1.10.48 to 1.11.812. * Introduce a new configuration parameter, dynamodb.region, which represents a valid AWS region code (see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions for example). If specifying the region, it is not necessary to specify the dynamodb.endpoint parameter. * Introduce support for non-standard endpoints. Before this commit, setting the endpoint to a non-standard one, like a proxy, but ultimately connecting to a region different from 'us-east-1' would lead to a credential errors due to bad signature: ERROR site.ycsb.db.DynamoDBClient -com.amazonaws.AmazonServiceException: Credential should be scoped to a valid region, not 'us-east-1'. (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: InvalidSignatureException; With this commit, if using a proxy as an endpoint, by setting also the region via dynamodb.region, it will work with no error. * Set TCP Keep-Alive to true. Even it makes sense to be used by YCSB, it apparently doesn't improve performance notably given the connection reuse that the DynamoDB client is doing. * Update the example config file with the endpoint and region parameters and behavior. --- dynamodb/conf/dynamodb.properties | 12 ++++-- dynamodb/pom.xml | 2 +- .../java/site/ycsb/db/DynamoDBClient.java | 39 +++++++++++++------ 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/dynamodb/conf/dynamodb.properties b/dynamodb/conf/dynamodb.properties index 691e20e80c..666f33dbe1 100644 --- a/dynamodb/conf/dynamodb.properties +++ b/dynamodb/conf/dynamodb.properties @@ -68,9 +68,15 @@ #when testing in HASH_AND_RANG mode. #dynamodb.hashKeyValue = -# Endpoint to connect to.For best latency, it is recommended -# to choose the endpoint which is closer to the client. -# Default is us-east-1 +# AWS Region code to connect to: +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions +# Set this parameter, unless you are using the default value ('us-east-1). +#dynamodb.region = us-east-1 + +# Endpoint to connect to. If not set, the endpoint will be set automatically +# based on the region and for HTTP connections. When using a non-standard +# endpoint (such as a proxy), the region parameter is still required to generate +# the proper message's signature. #dynamodb.endpoint = http://dynamodb.us-east-1.amazonaws.com # Strongly recommended to set to uniform.Refer FAQs in README diff --git a/dynamodb/pom.xml b/dynamodb/pom.xml index cd221889d8..824b9c979a 100644 --- a/dynamodb/pom.xml +++ b/dynamodb/pom.xml @@ -33,7 +33,7 @@ LICENSE file. com.amazonaws aws-java-sdk - 1.10.48 + 1.11.812 log4j diff --git a/dynamodb/src/main/java/site/ycsb/db/DynamoDBClient.java b/dynamodb/src/main/java/site/ycsb/db/DynamoDBClient.java index aabb5731a9..0ee7077530 100644 --- a/dynamodb/src/main/java/site/ycsb/db/DynamoDBClient.java +++ b/dynamodb/src/main/java/site/ycsb/db/DynamoDBClient.java @@ -19,13 +19,15 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.PropertiesCredentials; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import com.amazonaws.services.dynamodbv2.model.*; -import site.ycsb.*; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import site.ycsb.*; import java.io.File; import java.util.HashMap; @@ -35,7 +37,7 @@ import java.util.Vector; /** - * DynamoDB v1.10.48 client for YCSB. + * DynamoDB client for YCSB. */ public class DynamoDBClient extends DB { @@ -52,7 +54,7 @@ private enum PrimaryKeyType { HASH_AND_RANGE } - private AmazonDynamoDBClient dynamoDB; + private AmazonDynamoDB dynamoDB; private String primaryKeyName; private PrimaryKeyType primaryKeyType = PrimaryKeyType.HASH; @@ -63,7 +65,8 @@ private enum PrimaryKeyType { private String hashKeyName; private boolean consistentRead = false; - private String endpoint = "http://dynamodb.us-east-1.amazonaws.com"; + private String region = "us-east-1"; + private String endpoint = null; private int maxConnects = 50; private static final Logger LOGGER = Logger.getLogger(DynamoDBClient.class); private static final Status CLIENT_ERROR = new Status("CLIENT_ERROR", "An error occurred on the client."); @@ -83,6 +86,7 @@ public void init() throws DBException { String primaryKeyTypeString = getProperties().getProperty("dynamodb.primaryKeyType", null); String consistentReads = getProperties().getProperty("dynamodb.consistentReads", null); String connectMax = getProperties().getProperty("dynamodb.connectMax", null); + String configuredRegion = getProperties().getProperty("dynamodb.region", null); if (null != connectMax) { this.maxConnects = Integer.parseInt(connectMax); @@ -123,12 +127,25 @@ public void init() throws DBException { this.hashKeyValue = getProperties().getProperty("dynamodb.hashKeyValue", DEFAULT_HASH_KEY_VALUE); } + if (null != configuredRegion && configuredRegion.length() > 0) { + region = configuredRegion; + } + try { - AWSCredentials credentials = new PropertiesCredentials(new File(credentialsFile)); - ClientConfiguration cconfig = new ClientConfiguration(); - cconfig.setMaxConnections(maxConnects); - dynamoDB = new AmazonDynamoDBClient(credentials, cconfig); - dynamoDB.setEndpoint(this.endpoint); + AmazonDynamoDBClientBuilder dynamoDBBuilder = AmazonDynamoDBClientBuilder.standard(); + dynamoDBBuilder = null == endpoint ? + dynamoDBBuilder.withRegion(this.region) : + dynamoDBBuilder.withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(this.endpoint, this.region) + ); + dynamoDB = dynamoDBBuilder + .withClientConfiguration( + new ClientConfiguration() + .withTcpKeepAlive(true) + .withMaxConnections(this.maxConnects) + ) + .withCredentials(new AWSStaticCredentialsProvider(new PropertiesCredentials(new File(credentialsFile)))) + .build(); primaryKeyName = primaryKey; LOGGER.info("dynamodb connection created with " + this.endpoint); } catch (Exception e1) {