DynamoDB plugin

README.md

@@ -13,11 +13,12 @@
 ## Client APIs
 * Apache Cassandra (Thrift and CQL)
 * Redis
-* Netflix Dynomite (Redis API)
 * Elasticsearch
 * Elassandra
-* Geode
-* JanusGraph
+* AWS DynamoDB
+* Apache Geode
+* Apache JanusGraph
+* Netflix Dynomite (Redis API)
 * Netflix EVCache (Memcache API)
 
 ## Features

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Fri Sep 01 15:55:55 PDT 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-bin.zip

gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -154,16 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
   cd "$(dirname "$0")"
 fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

ndbench-dynamodb-plugins/build.gradle

@@ -0,0 +1,4 @@
+dependencies {
+    compile project(':ndbench-api')
+    compile "com.amazonaws:aws-java-sdk:latest.release"
+}

ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java

@@ -0,0 +1,233 @@
+/*
+ *  Copyright 2018 Netflix, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package com.netflix.ndbench.plugin.dynamodb;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
+import com.amazonaws.services.dynamodbv2.document.DynamoDB;
+import com.amazonaws.services.dynamodbv2.document.Item;
+import com.amazonaws.services.dynamodbv2.document.PutItemOutcome;
+import com.amazonaws.services.dynamodbv2.document.Table;
+import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
+import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
+import com.amazonaws.services.dynamodbv2.model.DescribeTableRequest;
+import com.amazonaws.services.dynamodbv2.model.GlobalTable;
+import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
+import com.amazonaws.services.dynamodbv2.model.KeyType;
+import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
+import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
+import com.amazonaws.services.dynamodbv2.model.TableDescription;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.ndbench.api.plugin.DataGenerator;
+import com.netflix.ndbench.api.plugin.NdBenchClient;
+import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPlugin;
+import com.netflix.ndbench.api.plugin.common.NdBenchConstants;
+import com.netflix.ndbench.plugin.dynamodb.configs.DynamoDBConfigs;
+
+/**
+ * This NDBench plugin provides a single key value for AWS DynamoDB.
+ * 
+ * @author ipapapa
+ */
+@Singleton
+@NdBenchClientPlugin("DynamoDBKeyValue")
+public class DynamoDBKeyValue implements NdBenchClient {
+	private final Logger logger = LoggerFactory.getLogger(DynamoDBKeyValue.class);
+	private static AmazonDynamoDB client;
+	private static DynamoDB dynamoDB;
+	private static AWSCredentialsProvider awsCredentialsProvider;
+	private DynamoDBConfigs config;
+	private static Table table;
+
+	private DataGenerator dataGenerator;
+
+	/**
+	 * Credentials will be loaded based on the environment. In AWS, the credentials
+	 * are based on the instance. In a local deployment they will have to provided.
+	 */
+	@Inject
+	public DynamoDBKeyValue(AWSCredentialsProvider credential, DynamoDBConfigs config, DataGenerator dataGenerator) {
+		this.config = config;
+		if (System.getenv(NdBenchConstants.DISCOVERY_ENV).equals("AWS")) {
+			awsCredentialsProvider = credential;
+		} else {
+			/*
+			 * The ProfileCredentialsProvider will return your [default] credential profile
+			 * by reading from the credentials file located at
+			 * (/home/username/.aws/credentials).
+			 */
+			awsCredentialsProvider = new ProfileCredentialsProvider();
+			try {
+				awsCredentialsProvider.getCredentials();
+			} catch (Exception e) {
+				throw new AmazonClientException("Cannot load the credentials from the credential profiles file. "
+						+ "Please make sure that your credentials file is at the correct "
+						+ "location (/home/<username>/.aws/credentials), and is in valid format.", e);
+			}
+		}
+	}
+
+	@Override
+	public void init(DataGenerator dataGenerator) throws Exception {
+		logger.info("Initing DynamoDB plugin");
+		client = AmazonDynamoDBClientBuilder.standard().withCredentials(awsCredentialsProvider).build();
+		dynamoDB = new DynamoDB(client);
+
+		/*
+		 * Create a table with a primary hash key named 'name', which holds a string.
+		 * Several properties such as provisioned throughput and atribute names are
+		 * defined in the configuration interface.
+		 */
+
+		logger.debug("Creating table if it does not exist yet");
+		table = dynamoDB.createTable(this.config.getTableName(),
+				Arrays.asList(new KeySchemaElement(config.getAttributeName(), KeyType.HASH)),
+				Arrays.asList(new AttributeDefinition("Id", ScalarAttributeType.N),
+						new AttributeDefinition("value", ScalarAttributeType.S)),
+				new ProvisionedThroughput(this.config.getReadCapacityUnits(), this.config.getWriteCapacityUnits()));
+
+		logger.debug("Waiting until the table is in ACTIVE state");
+		table.waitForActive();
+
+		DescribeTableRequest describeTableRequest = new DescribeTableRequest()
+				.withTableName(this.config.getTableName());
+		TableDescription tableDescription = client.describeTable(describeTableRequest).getTable();
+		logger.info("Table Description: " + tableDescription);
+
+		logger.info("DynamoDB Plugin initialized");
+	}
+
+	/**
+	 * 
+	 * @param key
+	 * @return the item
+	 * @throws Exception
+	 */
+	@Override
+	public String readSingle(String key) throws Exception {
+		Item item = null;
+		try {
+			GetItemSpec spec = new GetItemSpec().withPrimaryKey("Id", key).withConsistentRead(config.consistentRead());
+			item = table.getItem(spec);
+			if (item == null) {
+				return null;
+			}
+		} catch (AmazonServiceException ase) {
+			amazonServiceException(ase);
+		} catch (AmazonClientException ace) {
+			amazonClientException(ace);
+		}
+		return item.toString();
+	}
+
+	/**
+	 * 
+	 * @param key
+	 * @return A string representation of the output of a PutItemOutcome operation.
+	 * @throws Exception
+	 */
+	@Override
+	public String writeSingle(String key) throws Exception {
+		PutItemOutcome outcome = null;
+		try {
+			Item item = new Item().withPrimaryKey("Id", key).withString("value", this.dataGenerator.getRandomValue());
+			// Write the item to the table
+			outcome = table.putItem(item);
+			if (outcome == null) {
+				return null;
+			}
+
+		} catch (AmazonServiceException ase) {
+			amazonServiceException(ase);
+		} catch (AmazonClientException ace) {
+			amazonClientException(ace);
+		}
+		return outcome.toString();
+	}
+
+	@Override
+	public List<String> readBulk(List<String> keys) throws Exception {
+		return null;
+	}
+
+	@Override
+	public List<String> writeBulk(List<String> keys) throws Exception {
+		return null;
+	}
+
+	@Override
+	public void shutdown() throws Exception {
+		try {
+			logger.info("Issuing DeleteTable request for " + config.getTableName());
+			table.delete();
+
+			logger.info("Waiting for " + config.getTableName() + " to be deleted...this may take a while...");
+
+			table.waitForDelete();
+		} catch (Exception e) {
+			logger.error("DeleteTable request failed for " + config.getTableName());
+			logger.error(e.getMessage());
+		}
+		table.delete(); // cleanup
+		client.shutdown();
+		logger.info("DynamoDB shutdown");
+	}
+
+	/*
+	 * Not needed for this plugin
+	 * 
+	 * @see com.netflix.ndbench.api.plugin.NdBenchClient#getConnectionInfo()
+	 */
+	@Override
+	public String getConnectionInfo() throws Exception {
+		return null;
+	}
+
+	@Override
+	public String runWorkFlow() throws Exception {
+		return null;
+	}
+
+	private void amazonServiceException(AmazonServiceException ase) {
+
+		logger.error("Caught an AmazonServiceException, which means your request made it "
+				+ "to AWS, but was rejected with an error response for some reason.");
+		logger.error("Error Message:    " + ase.getMessage());
+		logger.error("HTTP Status Code: " + ase.getStatusCode());
+		logger.error("AWS Error Code:   " + ase.getErrorCode());
+		logger.error("Error Type:       " + ase.getErrorType());
+		logger.error("Request ID:       " + ase.getRequestId());
+	}
+
+	private void amazonClientException(AmazonClientException ace) {
+		logger.error("Caught an AmazonClientException, which means the client encountered "
+				+ "a serious internal problem while trying to communicate with AWS, "
+				+ "such as not being able to access the network.");
+		logger.error("Error Message: " + ace.getMessage());
+	}
+}

ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigs.java

@@ -0,0 +1,42 @@
+package com.netflix.ndbench.plugin.dynamodb.configs;
+
+import com.netflix.archaius.api.annotations.Configuration;
+import com.netflix.archaius.api.annotations.DefaultValue;
+
+/**
+ * Configurations for DynamoDB benchmarks
+ *
+ * @author ipapapa
+ */
+@Configuration(prefix = "ndbench.config.dynamodb")
+public interface DynamoDBConfigs {
+
+	@DefaultValue("ndbench-table")
+	String getTableName();
+
+	/*
+	 * Attributes – Each item is composed of one or more attributes. An attribute is
+	 * a fundamental data element, something that does not need to be broken down
+	 * any further.
+	 */
+	@DefaultValue("name")
+	String getAttributeName();
+
+	/*
+	 * Used for provisioned throughput
+	 */
+	@DefaultValue("1L")
+	Long getReadCapacityUnits();
+
+	@DefaultValue("1L")
+	Long getWriteCapacityUnits();
+
+	/*
+	 * Consistency: When you request a strongly consistent read, DynamoDB returns a
+	 * response with the most up-to-date data, reflecting the updates from all prior
+	 * write operations that were successful.
+	 */
+	@DefaultValue("false")
+	Boolean consistentRead();
+
+}

ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/IJanusGraphBuilder.java

@@ -1,3 +1,19 @@
+/*
+ *  Copyright 2018 Netflix, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
 package com.netflix.ndbench.plugins.janusgraph;
 
 import org.janusgraph.core.JanusGraphFactory;

ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphBasePlugin.java

@@ -1,3 +1,19 @@
+/*
+ *  Copyright 2018 Netflix, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
 package com.netflix.ndbench.plugins.janusgraph;
 
 import org.apache.tinkerpop.gremlin.structure.Vertex;