Add support for Geode and Cloud Foundry (#32)

* Add support for Geode and Cloud Foundry

 Updated
 1. Provided new discovery implementation for Cloud Foundry. Now you can
 push NdBench on Cloud Foundry and run.
 2. Added a new plugin for Geode in `local` and `cloud` mode.
 3. Updated read me on how to run NdBench on Cloud Foundry.

* Add support for Geode and Cloud Foundry

 Updated
 1. Provided new discovery implementation for Cloud Foundry. Now you can
 push NdBench on Cloud Foundry and run.
 2. Added a new plugin for Geode in `local` and `cloud` mode.
 3. Updated read me on how to run NdBench on Cloud Foundry.

* remove redundant @Inject on field which is set in constructor that also has an @Inject

* use logger, not std out for RPS/WPS info

* emit warning if expected read or write rate less than observed RPS (benchmark client could be the bottleneck)

* don't emit warning msgs until operations have started.

* add clarification to title of Instance Statistics dialog [ Statistics gathered per node, not aggregated across client driver instances ]

* remove newline

* api and driver support for auto-tuning write operations during benchmark run.

pls see documentation in com.netflix.ndbench.api.plugin.NdBenchAbstractClient for more details

* adding auto-tune support

* * add documentation to Json rendered NdBenchMonitor instances to explain what units are used for the metrics
* use settableConfig upon autoTune-initiated  change of writeRateLimit
* emit warn msg if Observed write/Read RPS  less than expected write or read rate
* augment IConfiguration interface with autoTune related config settings:

* fix merge errors

* 1) address comments on PR.
2) add unit test of RPSCount
    -- added code from supporting library for verifying log4j activity -- by just copying it in (since the library is not yet available in maven binary repositories).

* test one more case of rps count logger behavior

* * new unit test for WriteOperation.process()
* make constructor for NdBenchDriver package scope so test can instantiate an instance.

* Updates ndbench-geode-plugin with tests, allows DISCOVERY_ENV to set
ClusterDiscoveryService Implementation

Signed-off-by: Brenda Chan <brchan@pivotal.io>

.gitignore

@@ -5,3 +5,8 @@ build
 .project
 .settings
 bin/
+*.iml
+*.ipr
+*.iws
+.idea/*
+*.log

README.md

@@ -16,6 +16,7 @@
 * Dynomite with Redis
 * Elasticsearch
 * Elassandra
+* Geode
 
 ## Features
    * Dynamically change the benchmark configurations while the test is running, hence perform tests along with our production microservices.
@@ -24,6 +25,7 @@
    * Provide pluggable patterns and loads.
    * Support different client APIs.
    * Deploy, manage and monitor multiple instances from a single entry point.
+   * Support multiple cloud platform due to integration with Cloud Foundry
 
 Details about the features can be found in the [Wiki](https://github.com/Netflix/ndbench/wiki)
 
@@ -51,6 +53,11 @@ The first step before building ndbench is to configure the interfaces related to
 2. Set up Auto-Scale Group (ASG) and spin up instances
 3. Deploy `ndbench-web.war` in your container
 
+#### Deploy to Cloud Foundry
+
+1. Build and upload ndBench war to Cloud Foundry using `cf push`
+2. Note that the in the cf manifest that the `DISCOVERY_ENV` is set to `CF` so we can use the CfClusterDiscovery class
+
 ## Run
 	    ./gradlew appRun
 

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip

manifest.yml

@@ -0,0 +1,7 @@
+---
+applications:
+- name: ndbench
+  path: ndbench-web/build/libs/ndbench-web-0.4.0-SNAPSHOT.war
+  health-check-type: none
+  env:
+    DISCOVERY_ENV: CF

ndbench-api/src/main/java/com/netflix/ndbench/api/plugin/common/NdBenchConstants.java

@@ -49,4 +49,9 @@ public final class NdBenchConstants {
     public static final String WRITE_RATE_LIMIT_FULL_NAME = PROP_PREFIX + WRITE_RATE_LIMIT;
 
     public static final String CONFIG_CLUSTER_DISCOVERY_NAME="clusters.json";
+
+    public static final String DISCOVERY_ENV="DISCOVERY_ENV";
+    public static final String DISCOVERY_ENV_CF="CF";
+    public static final String DISCOVERY_ENV_AWS="AWS";
+
 }

ndbench-core/build.gradle

@@ -6,7 +6,7 @@ dependencies {
     compile 'com.sun.jersey:jersey-bundle:1.19.1'
     compile 'com.sun.jersey.contribs:jersey-multipart:1.19.1'
     compile 'com.sun.jersey.contribs:jersey-guice:1.19.1'
-
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.8.9'
 
     // For getting all implementations of an Interface
     compile group: 'org.reflections', name: 'reflections', version: '0.9.10'

ndbench-core/src/main/java/com/netflix/ndbench/core/defaultimpl/NdBenchGuiceModule.java

@@ -21,15 +21,16 @@
 import com.netflix.archaius.ConfigProxyFactory;
 import com.netflix.ndbench.api.plugin.DataGenerator;
 import com.netflix.ndbench.api.plugin.NdBenchMonitor;
+import com.netflix.ndbench.api.plugin.common.NdBenchConstants;
 import com.netflix.ndbench.core.config.IConfiguration;
 import com.netflix.ndbench.core.config.NdbenchConfigListener;
+import com.netflix.ndbench.core.discovery.AWSLocalClusterDiscovery;
+import com.netflix.ndbench.core.discovery.CfClusterDiscovery;
 import com.netflix.ndbench.core.discovery.IClusterDiscovery;
 import com.netflix.ndbench.core.discovery.LocalClusterDiscovery;
 import com.netflix.ndbench.core.generators.DefaultDataGenerator;
 import com.netflix.ndbench.core.monitoring.FakeMonitor;
 
-//import com.netflix.ndbench.core.config.NdBenchConfiguration;
-
 /**
  * @author vchella
  */
@@ -39,8 +40,14 @@ public class NdBenchGuiceModule extends AbstractModule
     protected void configure()
     {
         bind(NdBenchMonitor.class).to(FakeMonitor.class);
-        //bind(IClusterDiscovery.class).to(AWSLocalClusterDiscovery.class);
-        bind(IClusterDiscovery.class).to(LocalClusterDiscovery.class);
+        String discoveryEnv = System.getenv(NdBenchConstants.DISCOVERY_ENV);
+        if(discoveryEnv != null && discoveryEnv.equals(NdBenchConstants.DISCOVERY_ENV_CF)){
+            bind(IClusterDiscovery.class).to(CfClusterDiscovery.class);
+        }else if(discoveryEnv != null && discoveryEnv.equals(NdBenchConstants.DISCOVERY_ENV_AWS)){
+            bind(IClusterDiscovery.class).to(AWSLocalClusterDiscovery.class);
+        }else{
+            bind(IClusterDiscovery.class).to(LocalClusterDiscovery.class);
+        }
         bind(DataGenerator.class).to(DefaultDataGenerator.class);
         bind(NdbenchConfigListener.class).asEagerSingleton();
 

ndbench-core/src/main/java/com/netflix/ndbench/core/discovery/CfClusterDiscovery.java

@@ -0,0 +1,60 @@
+/*
+ *  Copyright 2016 Netflix, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package com.netflix.ndbench.core.discovery;
+
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author Pulkit Chandra
+ */
+public class CfClusterDiscovery implements IClusterDiscovery {
+    private static final Logger logger = LoggerFactory.getLogger(CfClusterDiscovery.class.getName());
+
+    @Override
+    public List<String> getApps() {
+        return Arrays.asList(getVmRouteName());
+    }
+
+    private String getVmRouteName() {
+        String vcap_application = System.getenv("VCAP_APPLICATION");
+        ObjectMapper mapper = new ObjectMapper();
+        Map<String, List<String>> vcap_map = new HashMap<>();
+        try {
+            vcap_map = mapper.readValue(vcap_application.getBytes(), HashMap.class);
+        } catch (IOException e) {
+            logger.error("Exception while reading vcap_application to Map" + e);
+        }
+        List<String> uris = vcap_map.get("uris");
+
+        return uris.get(0);
+    }
+
+    @Override
+    public List<String> getEndpoints(String appName) {
+        return Arrays.asList(getVmRouteName());
+    }
+
+}

ndbench-core/src/main/java/com/netflix/ndbench/core/generators/DefaultDataGenerator.java

@@ -18,6 +18,7 @@
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+
 import com.netflix.ndbench.api.plugin.DataGenerator;
 import com.netflix.ndbench.core.config.IConfiguration;
 import org.apache.commons.lang.RandomStringUtils;

ndbench-geode-plugins/build.gradle

@@ -0,0 +1,8 @@
+dependencies {
+    // We have dependency on the main project.
+    compile project(':ndbench-api')
+
+    testCompile 'org.mockito:mockito-all:1.10.19'
+    compile 'org.springframework.data:spring-data-geode:1.0.0.INCUBATING-RELEASE'
+
+}

ndbench-geode-plugins/src/main/java/com/netflix/ndbench/geode/plugin/ClientAuthInitialize.java

@@ -0,0 +1,41 @@
+package com.netflix.ndbench.geode.plugin;
+
+
+import org.apache.geode.LogWriter;
+import org.apache.geode.distributed.DistributedMember;
+import org.apache.geode.security.AuthInitialize;
+import org.apache.geode.security.AuthenticationFailedException;
+
+import java.util.Properties;
+
+/**
+ * Created by Pulkit Chandra
+ */
+@SuppressWarnings("unused")
+public class ClientAuthInitialize implements AuthInitialize {
+
+  public static final String USER_NAME = "security-username";
+  public static final String PASSWORD = "security-password";
+
+  public static AuthInitialize create() {
+    return new ClientAuthInitialize();
+  }
+
+  @Override
+  public void close() {
+  }
+
+  @Override
+  public Properties getCredentials(Properties arg0, DistributedMember arg1,
+                                   boolean arg2) throws AuthenticationFailedException {
+    Properties props = new Properties();
+    props.put(USER_NAME, arg0.getProperty(USER_NAME));
+    props.put(PASSWORD, arg0.getProperty(PASSWORD));
+    return props;
+  }
+
+  @Override
+  public void init(LogWriter arg0, LogWriter arg1)
+      throws AuthenticationFailedException {
+  }
+}

ndbench-geode-plugins/src/main/java/com/netflix/ndbench/geode/plugin/EnvParser.java

@@ -0,0 +1,88 @@
+package com.netflix.ndbench.geode.plugin;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.archaius.api.PropertyFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Reused from
+ * https://github.com/gemfire/cf-gemfire-connector-examples
+ *
+ * @author Pulkit Chandra
+ */
+@Singleton
+public class EnvParser {
+  private final static Logger logger = LoggerFactory.getLogger(EnvParser.class);
+  private final String PROPERTIES_VCAP_SERVICES = "VCAP_SERVICES";
+  private final String SERVICE_NAME = "p-cloudcache";
+  private PropertyFactory propertyFactory;
+
+  private final Pattern p = Pattern.compile("(.*)\\[(\\d*)\\]");
+
+  @Inject
+  public EnvParser(PropertyFactory propertyFactory) {
+    this.propertyFactory = propertyFactory;
+
+  }
+
+  public List<URI> getLocators() throws IOException, URISyntaxException {
+    List<URI> locatorList = new ArrayList<URI>();
+    Map credentials = getCredentials();
+    List<String> locators = (List<String>) credentials.get("locators");
+    for (String locator : locators) {
+      Matcher m = p.matcher(locator);
+      if (!m.matches()) {
+        throw new IllegalStateException("Unexpected locator format. expected host[port], got"+locator);
+      }
+      locatorList.add(new URI("locator://" + m.group(1) + ":" + m.group(2)));
+    }
+    return locatorList;
+  }
+
+  public String getUsername() throws IOException {
+    Map credentials = getCredentials();
+    List<Map<String, String>> userData = (List<Map<String, String>>) credentials.get("users");
+    return userData.get(0).get("username");
+  }
+
+
+  public String getPasssword() throws IOException {
+    Map credentials = getCredentials();
+    List<Map<String, String>> userData = (List<Map<String, String>>) credentials.get("users");
+    return userData.get(0).get("password");
+  }
+
+  private Map getCredentials() throws IOException {
+    Map credentials = null;
+    String envContent = propertyFactory.getProperty(PROPERTIES_VCAP_SERVICES).asString("{}").get();
+    ObjectMapper objectMapper = new ObjectMapper();
+    Map services = objectMapper.readValue(envContent, Map.class);
+    List gemfireService = getGemFireService(services);
+    if (gemfireService != null) {
+      Map serviceInstance = (Map) gemfireService.get(0);
+      credentials = (Map) serviceInstance.get("credentials");
+    }
+    return credentials;
+
+  }
+
+  private List getGemFireService(Map services) {
+    List l = (List) services.get(SERVICE_NAME);
+    if (l == null) {
+      throw new IllegalStateException("GemFire service is not bound to this application");
+    }
+    return l;
+  }
+}