diff --git a/README.md b/README.md index c10e1410..f78c939f 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,11 @@ ## Client APIs * Apache Cassandra (Thrift and CQL) * Redis +* Netflix Dynomite (Redis API) * Elasticsearch * Elassandra -* AWS DynamoDB -* Apache Geode -* Apache JanusGraph -* Netflix Dynomite (Redis API) +* Geode +* JanusGraph * Netflix EVCache (Memcache API) ## Features diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7a3265ee..6ffa2378 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 568c50bf..eb1ac183 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ +#Fri Sep 01 15:55:55 PDT 2017 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip diff --git a/gradlew b/gradlew index cccdd3d5..9aa616c2 100755 --- a/gradlew +++ b/gradlew @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash ############################################################################## ## @@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS="" # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" -warn () { +warn ( ) { echo "$*" } -die () { +die ( ) { echo echo "$*" echo @@ -154,19 +154,16 @@ if $cygwin ; then esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") } -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then +if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then cd "$(dirname "$0")" fi -exec "$JAVACMD" "$@" +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/ndbench-cass-plugins/src/main/java/com/netflix/ndbench/plugin/cass/CassJavaDriverPlugin.java b/ndbench-cass-plugins/src/main/java/com/netflix/ndbench/plugin/cass/CassJavaDriverPlugin.java new file mode 100644 index 00000000..e69de29b diff --git a/ndbench-dynamodb-plugins/build.gradle b/ndbench-dynamodb-plugins/build.gradle deleted file mode 100644 index 0fe53472..00000000 --- a/ndbench-dynamodb-plugins/build.gradle +++ /dev/null @@ -1,4 +0,0 @@ -dependencies { - compile project(':ndbench-api') - compile "com.amazonaws:aws-java-sdk:latest.release" -} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java deleted file mode 100644 index bd47041b..00000000 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.ndbench.plugin.dynamodb; - -import java.util.Arrays; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; -import com.amazonaws.services.dynamodbv2.document.DynamoDB; -import com.amazonaws.services.dynamodbv2.document.Item; -import com.amazonaws.services.dynamodbv2.document.PutItemOutcome; -import com.amazonaws.services.dynamodbv2.document.Table; -import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec; -import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; -import com.amazonaws.services.dynamodbv2.model.DescribeTableRequest; -import com.amazonaws.services.dynamodbv2.model.GlobalTable; -import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; -import com.amazonaws.services.dynamodbv2.model.KeyType; -import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; -import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; -import com.amazonaws.services.dynamodbv2.model.TableDescription; -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.netflix.ndbench.api.plugin.DataGenerator; -import com.netflix.ndbench.api.plugin.NdBenchClient; -import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPlugin; -import com.netflix.ndbench.api.plugin.common.NdBenchConstants; -import com.netflix.ndbench.plugin.dynamodb.configs.DynamoDBConfigs; - -/** - * This NDBench plugin provides a single key value for AWS DynamoDB. - * - * @author ipapapa - */ -@Singleton -@NdBenchClientPlugin("DynamoDBKeyValue") -public class DynamoDBKeyValue implements NdBenchClient { - private final Logger logger = LoggerFactory.getLogger(DynamoDBKeyValue.class); - private static AmazonDynamoDB client; - private static DynamoDB dynamoDB; - private static AWSCredentialsProvider awsCredentialsProvider; - private DynamoDBConfigs config; - private static Table table; - - private DataGenerator dataGenerator; - - /** - * Credentials will be loaded based on the environment. In AWS, the credentials - * are based on the instance. In a local deployment they will have to provided. - */ - @Inject - public DynamoDBKeyValue(AWSCredentialsProvider credential, DynamoDBConfigs config, DataGenerator dataGenerator) { - this.config = config; - if (System.getenv(NdBenchConstants.DISCOVERY_ENV).equals("AWS")) { - awsCredentialsProvider = credential; - } else { - /* - * The ProfileCredentialsProvider will return your [default] credential profile - * by reading from the credentials file located at - * (/home/username/.aws/credentials). - */ - awsCredentialsProvider = new ProfileCredentialsProvider(); - try { - awsCredentialsProvider.getCredentials(); - } catch (Exception e) { - throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " - + "Please make sure that your credentials file is at the correct " - + "location (/home//.aws/credentials), and is in valid format.", e); - } - } - } - - @Override - public void init(DataGenerator dataGenerator) throws Exception { - logger.info("Initing DynamoDB plugin"); - client = AmazonDynamoDBClientBuilder.standard().withCredentials(awsCredentialsProvider).build(); - dynamoDB = new DynamoDB(client); - - /* - * Create a table with a primary hash key named 'name', which holds a string. - * Several properties such as provisioned throughput and atribute names are - * defined in the configuration interface. - */ - - logger.debug("Creating table if it does not exist yet"); - table = dynamoDB.createTable(this.config.getTableName(), - Arrays.asList(new KeySchemaElement(config.getAttributeName(), KeyType.HASH)), - Arrays.asList(new AttributeDefinition("Id", ScalarAttributeType.N), - new AttributeDefinition("value", ScalarAttributeType.S)), - new ProvisionedThroughput(this.config.getReadCapacityUnits(), this.config.getWriteCapacityUnits())); - - logger.debug("Waiting until the table is in ACTIVE state"); - table.waitForActive(); - - DescribeTableRequest describeTableRequest = new DescribeTableRequest() - .withTableName(this.config.getTableName()); - TableDescription tableDescription = client.describeTable(describeTableRequest).getTable(); - logger.info("Table Description: " + tableDescription); - - logger.info("DynamoDB Plugin initialized"); - } - - /** - * - * @param key - * @return the item - * @throws Exception - */ - @Override - public String readSingle(String key) throws Exception { - Item item = null; - try { - GetItemSpec spec = new GetItemSpec().withPrimaryKey("Id", key).withConsistentRead(config.consistentRead()); - item = table.getItem(spec); - if (item == null) { - return null; - } - } catch (AmazonServiceException ase) { - amazonServiceException(ase); - } catch (AmazonClientException ace) { - amazonClientException(ace); - } - return item.toString(); - } - - /** - * - * @param key - * @return A string representation of the output of a PutItemOutcome operation. - * @throws Exception - */ - @Override - public String writeSingle(String key) throws Exception { - PutItemOutcome outcome = null; - try { - Item item = new Item().withPrimaryKey("Id", key).withString("value", this.dataGenerator.getRandomValue()); - // Write the item to the table - outcome = table.putItem(item); - if (outcome == null) { - return null; - } - - } catch (AmazonServiceException ase) { - amazonServiceException(ase); - } catch (AmazonClientException ace) { - amazonClientException(ace); - } - return outcome.toString(); - } - - @Override - public List readBulk(List keys) throws Exception { - return null; - } - - @Override - public List writeBulk(List keys) throws Exception { - return null; - } - - @Override - public void shutdown() throws Exception { - try { - logger.info("Issuing DeleteTable request for " + config.getTableName()); - table.delete(); - - logger.info("Waiting for " + config.getTableName() + " to be deleted...this may take a while..."); - - table.waitForDelete(); - } catch (Exception e) { - logger.error("DeleteTable request failed for " + config.getTableName()); - logger.error(e.getMessage()); - } - table.delete(); // cleanup - client.shutdown(); - logger.info("DynamoDB shutdown"); - } - - /* - * Not needed for this plugin - * - * @see com.netflix.ndbench.api.plugin.NdBenchClient#getConnectionInfo() - */ - @Override - public String getConnectionInfo() throws Exception { - return null; - } - - @Override - public String runWorkFlow() throws Exception { - return null; - } - - private void amazonServiceException(AmazonServiceException ase) { - - logger.error("Caught an AmazonServiceException, which means your request made it " - + "to AWS, but was rejected with an error response for some reason."); - logger.error("Error Message: " + ase.getMessage()); - logger.error("HTTP Status Code: " + ase.getStatusCode()); - logger.error("AWS Error Code: " + ase.getErrorCode()); - logger.error("Error Type: " + ase.getErrorType()); - logger.error("Request ID: " + ase.getRequestId()); - } - - private void amazonClientException(AmazonClientException ace) { - logger.error("Caught an AmazonClientException, which means the client encountered " - + "a serious internal problem while trying to communicate with AWS, " - + "such as not being able to access the network."); - logger.error("Error Message: " + ace.getMessage()); - } -} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigs.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigs.java deleted file mode 100644 index a329d8ef..00000000 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigs.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.netflix.ndbench.plugin.dynamodb.configs; - -import com.netflix.archaius.api.annotations.Configuration; -import com.netflix.archaius.api.annotations.DefaultValue; - -/** - * Configurations for DynamoDB benchmarks - * - * @author ipapapa - */ -@Configuration(prefix = "ndbench.config.dynamodb") -public interface DynamoDBConfigs { - - @DefaultValue("ndbench-table") - String getTableName(); - - /* - * Attributes – Each item is composed of one or more attributes. An attribute is - * a fundamental data element, something that does not need to be broken down - * any further. - */ - @DefaultValue("name") - String getAttributeName(); - - /* - * Used for provisioned throughput - */ - @DefaultValue("1L") - Long getReadCapacityUnits(); - - @DefaultValue("1L") - Long getWriteCapacityUnits(); - - /* - * Consistency: When you request a strongly consistent read, DynamoDB returns a - * response with the most up-to-date data, reflecting the updates from all prior - * write operations that were successful. - */ - @DefaultValue("false") - Boolean consistentRead(); - -} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBModule.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBModule.java deleted file mode 100644 index f86fb15b..00000000 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBModule.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.netflix.ndbench.plugin.dynamodb.configs; - -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.netflix.archaius.ConfigProxyFactory; -import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPluginGuiceModule; - -/** - * - * @author ipapapa - * - */ -@NdBenchClientPluginGuiceModule -public class DynamoDBModule extends AbstractModule { - - @Override - protected void configure() { - } - - @Provides - DynamoDBConfigs getDynamoDBConfigs(ConfigProxyFactory factory) { - return factory.newProxy(DynamoDBConfigs.class); - } - -} \ No newline at end of file diff --git a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/IJanusGraphBuilder.java b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/IJanusGraphBuilder.java index a29106ed..1c39120a 100644 --- a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/IJanusGraphBuilder.java +++ b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/IJanusGraphBuilder.java @@ -1,19 +1,3 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ package com.netflix.ndbench.plugins.janusgraph; import org.janusgraph.core.JanusGraphFactory; diff --git a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphBasePlugin.java b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphBasePlugin.java index b8a1091b..5e59f1fd 100644 --- a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphBasePlugin.java +++ b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphBasePlugin.java @@ -1,19 +1,3 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ package com.netflix.ndbench.plugins.janusgraph; import org.apache.tinkerpop.gremlin.structure.Vertex; diff --git a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphPluginCQL.java b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphPluginCQL.java index b8bbea95..696b91d9 100644 --- a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphPluginCQL.java +++ b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/JanusGraphPluginCQL.java @@ -1,19 +1,3 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ package com.netflix.ndbench.plugins.janusgraph; import com.netflix.ndbench.api.plugin.DataGenerator; @@ -68,7 +52,6 @@ public void init(DataGenerator dataGenerator) throws Exception { this.traversalSource = graph.traversal(); this.dataGenerator = dataGenerator; createSchema(graph); - Logger.info("Initing JanusGraph Plugin CQL"); } @Override @@ -163,7 +146,6 @@ public List writeBulk(final List keys) throws Exception { @Override public void shutdown() throws Exception { graph.close(); - Logger.info("JanusGraph DB shutdown"); } @Override diff --git a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/NdBenchJanusGraphModule.java b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/NdBenchJanusGraphModule.java index 984c61de..ce45a688 100644 --- a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/NdBenchJanusGraphModule.java +++ b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/NdBenchJanusGraphModule.java @@ -1,19 +1,3 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ package com.netflix.ndbench.plugins.janusgraph; import com.google.inject.AbstractModule; diff --git a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/configs/IJanusGraphConfig.java b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/configs/IJanusGraphConfig.java index a5edda5d..36ac1acb 100644 --- a/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/configs/IJanusGraphConfig.java +++ b/ndbench-janusgraph-plugins/src/main/java/com/netflix/ndbench/plugins/janusgraph/configs/IJanusGraphConfig.java @@ -1,19 +1,3 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ package com.netflix.ndbench.plugins.janusgraph.configs; import com.netflix.archaius.api.annotations.Configuration; diff --git a/ndbench-web/src/main/java/com/netflix/ndbench/defaultimpl/InjectedWebListener.java b/ndbench-web/src/main/java/com/netflix/ndbench/defaultimpl/InjectedWebListener.java index 34984229..7f67df35 100644 --- a/ndbench-web/src/main/java/com/netflix/ndbench/defaultimpl/InjectedWebListener.java +++ b/ndbench-web/src/main/java/com/netflix/ndbench/defaultimpl/InjectedWebListener.java @@ -20,13 +20,19 @@ import com.google.inject.Injector; import com.google.inject.servlet.GuiceServletContextListener; import com.google.inject.servlet.ServletModule; +import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPlugin; +import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPluginGuiceModule; import com.netflix.ndbench.core.config.GuiceInjectorProvider; +import com.netflix.ndbench.plugin.es.NfndbenchEsModule; import com.sun.jersey.api.core.PackagesResourceConfig; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.spi.container.servlet.ServletContainer; +import com.netflix.ndbench.plugins.janusgraph.NdBenchJanusGraphModule; +import org.reflections.Reflections; import java.util.HashMap; import java.util.Map; +import java.util.Set; /** * @author vchella diff --git a/settings.gradle b/settings.gradle index 5fb616f3..5ba84117 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,3 @@ rootProject.name='ndbench' -include 'ndbench-web','ndbench-api','ndbench-core','ndbench-cass-plugins','ndbench-sample-plugins','ndbench-dyno-plugins','ndbench-es-plugins','ndbench-geode-plugins','ndbench-janusgraph-plugins', 'ndbench-evcache-plugins', 'ndbench-dynamodb-plugins' +include 'ndbench-web','ndbench-api','ndbench-core','ndbench-cass-plugins','ndbench-sample-plugins','ndbench-dyno-plugins','ndbench-es-plugins','ndbench-geode-plugins','ndbench-janusgraph-plugins', 'ndbench-evcache-plugins'