Permalink
Browse files

Add new consistent hash ring implementation which is more balanced an…

…d uses less memory

RB=772345
G=si-core-reviewers
R=dhoa,cxu
A=dhoa
  • Loading branch information...
1 parent 3578bf5 commit 319b651a4f2a4d763ec0085019cf4af9a15db55b Ang Xu committed Jul 25, 2016
Showing with 1,010 additions and 363 deletions.
  1. +3 −0 build.gradle
  2. +15 −0 d2-benchmark/build.gradle
  3. +132 −0 d2-benchmark/src/jmh/java/com/linkedin/d2/util/hashing/ConsistentHashRingBenchmark.java
  4. +23 −0 d2-schemas/src/main/pegasus/com/linkedin/d2/D2LoadBalancerStrategyProperties.pdsc
  5. +1 −0 d2/build.gradle
  6. +34 −1 d2/src/main/java/com/linkedin/d2/balancer/config/LoadBalancerStrategyPropertiesConverter.java
  7. +2 −0 d2/src/main/java/com/linkedin/d2/balancer/properties/PropertyKeys.java
  8. +34 −4 ...rc/main/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerStrategyConfig.java
  9. +17 −154 d2/src/main/java/com/linkedin/d2/balancer/strategies/degrader/DegraderRingFactory.java
  10. +43 −0 d2/src/main/java/com/linkedin/d2/balancer/strategies/degrader/MPConsistentHashRingFactory.java
  11. +195 −0 ...c/main/java/com/linkedin/d2/balancer/strategies/degrader/PointBasedConsistentHashRingFactory.java
  12. +198 −0 d2/src/main/java/com/linkedin/d2/balancer/util/hashing/MPConsistentHashRing.java
  13. +9 −1 d2/src/test/java/com/linkedin/d2/balancer/config/LoadBalancerStrategyPropertiesConverterTest.java
  14. +1 −1 d2/src/test/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerStateTest.java
  15. +9 −0 ...est/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerStrategyConfigTest.java
  16. +42 −141 d2/src/test/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerTest.java
  17. +3 −4 d2/src/test/java/com/linkedin/d2/balancer/strategies/degrader/DegraderRingFactoryTest.java
  18. +71 −51 d2/src/test/java/com/linkedin/d2/balancer/util/hashing/ConsistentHashKeyMapperTest.java
  19. +165 −0 d2/src/test/java/com/linkedin/d2/balancer/util/hashing/MPConsistentHashTest.java
  20. +12 −6 restli-int-test/src/test/java/com/linkedin/restli/examples/TestAllPartitionsRequestBuilder.java
  21. +1 −0 settings.gradle
View
@@ -51,6 +51,8 @@ project.ext.externalDependency = [
'javaxInject': 'javax.inject:javax.inject:1',
'jdkTools': files("${System.getProperty('java.home')}/../lib/tools.jar"),
'jetty': 'org.eclipse.jetty.aggregate:jetty-all:8.1.8.v20121106',
+ "jmhCore": "org.openjdk.jmh:jmh-core:1.11",
+ "jmhAnnotations": "org.openjdk.jmh:jmh-generator-annprocess:1.11",
'log4j2Api': 'org.apache.logging.log4j:log4j-api:2.0.2',
'log4j2Core': 'org.apache.logging.log4j:log4j-core:2.0.2',
'log4jLog4j2': 'org.apache.logging.log4j:log4j-1.2-api:2.0.2',
@@ -66,6 +68,7 @@ project.ext.externalDependency = [
'snappy': 'org.iq80.snappy:snappy:0.3',
'testng': 'org.testng:testng:6.9.9',
'velocity': 'org.apache.velocity:velocity:1.5',
+ 'zero_allocation_hashing': 'net.openhft:zero-allocation-hashing:0.4',
'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.6',
// for restli-spring-bridge ONLY, we must keep these dependencies isolated
@@ -0,0 +1,15 @@
+plugins {
+ id 'me.champeau.gradle.jmh' version '0.3.0'
+}
+
+jmh {
+ include = '.*ConsistentHashRingBenchmark.*'
+ zip64 = true
+}
+
+
+dependencies {
+ jmh project(':d2')
+ jmh externalDependency.jmhCore
+ jmh externalDependency.jmhAnnotations
+}
@@ -0,0 +1,132 @@
+/*
+ Copyright (c) 2016 LinkedIn Corp.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package com.linkedin.d2.util.hashing;
+
+import com.linkedin.d2.balancer.util.hashing.ConsistentHashRing;
+import com.linkedin.d2.balancer.util.hashing.MPConsistentHashRing;
+import com.linkedin.d2.balancer.util.hashing.Ring;
+import java.net.URI;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+
+/**
+ * @author Ang Xu
+ */
+@Fork(2)
+@Warmup(iterations = 5)
+@Measurement(iterations = 5)
+public class ConsistentHashRingBenchmark {
+
+ @State(Scope.Benchmark)
+ public static class MPCHash_10Hosts_11Probes_State {
+ Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(10, 100), 11);
+ Random _random = new Random();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SampleTime)
+ @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ public URI measureMPCHash_10Hosts_11Probes(MPCHash_10Hosts_11Probes_State state) {
+ return state._ring.get(state._random.nextInt());
+ }
+
+ @State(Scope.Benchmark)
+ public static class MPCHash_10Hosts_21Probes_State {
+ Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(10, 100), 21);
+ Random _random = new Random();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SampleTime)
+ @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ public URI measureMPCHash_10Hosts_21Probes(MPCHash_10Hosts_21Probes_State state) {
+ return state._ring.get(state._random.nextInt());
+ }
+
+ @State(Scope.Benchmark)
+ public static class MPCHash_100Hosts_11Probes_State {
+ Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(100, 100), 11);
+ Random _random = new Random();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SampleTime)
+ @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ public URI measureMPCHash_100Hosts_11Probes(MPCHash_100Hosts_11Probes_State state) {
+ return state._ring.get(state._random.nextInt());
+ }
+
+ @State(Scope.Benchmark)
+ public static class MPCHash_100Hosts_21Probes_State {
+ Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(100, 100), 21);
+ Random _random = new Random();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SampleTime)
+ @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ public URI measureMPCHash_100Hosts_21Probes(MPCHash_100Hosts_21Probes_State state) {
+ return state._ring.get(state._random.nextInt());
+ }
+
+ @State(Scope.Benchmark)
+ public static class ConsistentHashRing_10Hosts_100PointsPerHost_State {
+ Ring<URI> _ring = new ConsistentHashRing<URI>(buildPointsMap(10, 100));
+ Random _random = new Random();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SampleTime)
+ @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ public URI measureonsistentHashRing_10Hosts_100PointsPerHost(ConsistentHashRing_10Hosts_100PointsPerHost_State state) {
+ return state._ring.get(state._random.nextInt());
+ }
+
+
+ @State(Scope.Benchmark)
+ public static class ConsistentHashRing_100Hosts_100PointsPerHost_State {
+ Ring<URI> _ring = new ConsistentHashRing<URI>(buildPointsMap(100, 100));
+ Random _random = new Random();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SampleTime)
+ @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ public URI measureonsistentHashRing_100Hosts_100PointsPerHost(ConsistentHashRing_100Hosts_100PointsPerHost_State state) {
+ return state._ring.get(state._random.nextInt());
+ }
+
+ private static Map<URI, Integer> buildPointsMap(int numHosts, int numPointsPerHost) {
+ return IntStream.range(0, numHosts).boxed().collect(
+ Collectors.toMap(
+ key -> URI.create(String.format("app-%04d.linkedin.com", key)),
+ value -> numPointsPerHost));
+ }
+}
@@ -118,6 +118,29 @@
"type": "double",
"doc": "The highest ratio of unused entries over the total entries of the Ring points that d2 maintains.",
"optional": true
+ },
+ {
+ "name": "consistentHashAlgorithm",
+ "type": {
+ "name": "ConsistentHashAlgorithmEnum",
+ "type": "enum",
+ "symbols": [
+ "POINT_BASED",
+ "MULTI_PROBE"
+ ],
+ "symbolDocs": {
+ "POINT_BASED": "Point-based consistent hash ring. The more points the ring has, the more balanced it is.",
+ "MULTI_PROBE": "Multi-probe consistent hash. The more probes to use, the more balanced the ring is."
+ }
+ },
+ "doc": "Consistent hash algorithm the d2 load balancer should use. Defaults to POINT_BASED.",
+ "optional": true
+ },
+ {
+ "name": "numberOfProbes",
+ "type": "int",
+ "doc": "The number of probes used to look up a key in consistent hash ring. Defaults to 21.",
+ "optional": true
}
]
}
View
@@ -16,6 +16,7 @@ dependencies {
compile externalDependency.jacksonCore
compile externalDependency.jacksonDataBind
compile externalDependency.jdkTools
+ compile externalDependency.zero_allocation_hashing
testCompile externalDependency.testng
testCompile externalDependency.commonsIo
testCompile externalDependency.easymock
@@ -16,14 +16,15 @@
package com.linkedin.d2.balancer.config;
+import com.linkedin.d2.ConsistentHashAlgorithmEnum;
import com.linkedin.d2.D2LoadBalancerStrategyProperties;
import com.linkedin.d2.balancer.properties.PropertyKeys;
import com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3;
+import com.linkedin.d2.balancer.strategies.degrader.DegraderRingFactory;
import com.linkedin.d2.balancer.util.hashing.URIRegexHash;
import com.linkedin.d2.hashConfigType;
import com.linkedin.d2.hashMethodEnum;
import com.linkedin.data.template.StringArray;
-import com.linkedin.data.template.StringMap;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -126,6 +127,22 @@
{
map.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_ONLY_AT_INTERVAL, config.isUpdateOnlyAtInterval().toString());
}
+ if (config.hasConsistentHashAlgorithm())
+ {
+ switch (config.getConsistentHashAlgorithm())
+ {
+ case MULTI_PROBE:
+ map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DegraderRingFactory.MULTI_PROBE_CONSISTENT_HASH);
+ break;
+ case POINT_BASED:
+ map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DegraderRingFactory.POINT_BASED_CONSISTENT_HASH);
+ break;
+ }
+ }
+ if (config.hasNumberOfProbes())
+ {
+ map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES, config.getNumberOfProbes().toString());
+ }
return map;
}
@@ -218,6 +235,22 @@ else if (DegraderLoadBalancerStrategyV3.HASH_METHOD_URI_REGEX.equalsIgnoreCase(h
coerce(properties.get(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_ONLY_AT_INTERVAL),
Boolean.class));
}
+ if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM))
+ {
+ String consistentHashAlgorithm = coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM), String.class);
+ if (DegraderRingFactory.POINT_BASED_CONSISTENT_HASH.equalsIgnoreCase(consistentHashAlgorithm))
+ {
+ config.setConsistentHashAlgorithm(ConsistentHashAlgorithmEnum.POINT_BASED);
+ }
+ else if (DegraderRingFactory.MULTI_PROBE_CONSISTENT_HASH.equalsIgnoreCase(consistentHashAlgorithm))
+ {
+ config.setConsistentHashAlgorithm(ConsistentHashAlgorithmEnum.MULTI_PROBE);
+ }
+ }
+ if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES))
+ {
+ config.setNumberOfProbes(coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES), Integer.class));
+ }
return config;
}
}
@@ -84,6 +84,8 @@
public static final String HTTP_LB_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK = "http.loadBalancer.clusterMinCallCount.highWaterMark";
public static final String HTTP_LB_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK = "http.loadBalancer.clusterMinCallCount.lowWaterMark";
public static final String HTTP_LB_HASHRING_POINT_CLEANUP_RATE = "http.loadBalancer.hashRingPointCleanupRate";
+ public static final String HTTP_LB_CONSISTENT_HASH_ALGORITHM = "http.loadBalancer.consistentHashAlgorithm";
+ public static final String HTTP_LB_CONSISTENT_HASH_NUM_PROBES = "http.loadBalancer.consistentHashNumProbes";
//used by service metadata properties
public static final String SERVICE_FOLLOW_REDIRECTION_MAX_HOP = "followRedirection.maxHop";
@@ -17,6 +17,7 @@
package com.linkedin.d2.balancer.strategies.degrader;
import com.linkedin.d2.balancer.properties.PropertyKeys;
+import com.linkedin.d2.balancer.util.hashing.MPConsistentHashRing;
import java.util.Collections;
import java.util.Map;
@@ -64,6 +65,9 @@
private final double _hashRingPointCleanUpRate;
+ private final String _consistentHashAlgorithm;
+ private final int _numProbes;
+
public static final Clock DEFAULT_CLOCK = SystemClock.instance();
public static final double DEFAULT_INITIAL_RECOVERY_LEVEL = 0.01;
public static final double DEFAULT_RAMP_FACTOR = 1.0;
@@ -84,14 +88,17 @@
public static final double DEFAULT_HASHRING_POINT_CLEANUP_RATE = 0.20;
+ public static final int DEFAULT_NUM_PROBES = MPConsistentHashRing.DEFAULT_NUM_PROBES;
+
public DegraderLoadBalancerStrategyConfig(long updateIntervalMs)
{
this(updateIntervalMs, DEFAULT_UPDATE_ONLY_AT_INTERVAL, 100, null, Collections.<String, Object>emptyMap(),
DEFAULT_CLOCK, DEFAULT_INITIAL_RECOVERY_LEVEL, DEFAULT_RAMP_FACTOR, DEFAULT_HIGH_WATER_MARK, DEFAULT_LOW_WATER_MARK,
DEFAULT_GLOBAL_STEP_UP, DEFAULT_GLOBAL_STEP_DOWN,
DEFAULT_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK,
DEFAULT_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK,
- DEFAULT_HASHRING_POINT_CLEANUP_RATE);
+ DEFAULT_HASHRING_POINT_CLEANUP_RATE,
+ null, DEFAULT_NUM_PROBES);
}
public DegraderLoadBalancerStrategyConfig(DegraderLoadBalancerStrategyConfig config)
@@ -110,7 +117,9 @@ public DegraderLoadBalancerStrategyConfig(DegraderLoadBalancerStrategyConfig con
config.getGlobalStepDown(),
config.getMinClusterCallCountHighWaterMark(),
config.getMinClusterCallCountLowWaterMark(),
- config.getHashRingPointCleanUpRate());
+ config.getHashRingPointCleanUpRate(),
+ config.getConsistentHashAlgorithm(),
+ config.getNumProbes());
}
public DegraderLoadBalancerStrategyConfig(long updateIntervalMs,
@@ -127,7 +136,9 @@ public DegraderLoadBalancerStrategyConfig(long updateIntervalMs,
double globalStepDown,
long minCallCountHighWaterMark,
long minCallCountLowWaterMark,
- double hashRingPointCleanUpRate)
+ double hashRingPointCleanUpRate,
+ String consistentHashAlgorithm,
+ int numProbes)
{
_updateIntervalMs = updateIntervalMs;
_updateOnlyAtInterval = updateOnlyAtInterval;
@@ -144,6 +155,8 @@ public DegraderLoadBalancerStrategyConfig(long updateIntervalMs,
_minClusterCallCountHighWaterMark = minCallCountHighWaterMark;
_minClusterCallCountLowWaterMark = minCallCountLowWaterMark;
_hashRingPointCleanUpRate = hashRingPointCleanUpRate;
+ _consistentHashAlgorithm = consistentHashAlgorithm;
+ _numProbes = numProbes;
}
/**
@@ -214,11 +227,18 @@ public static DegraderLoadBalancerStrategyConfig createHttpConfigFromMap(Map<Str
Double hashRingPointCleanUpRate = MapUtil.getWithDefault(map, PropertyKeys.HTTP_LB_HASHRING_POINT_CLEANUP_RATE,
DEFAULT_HASHRING_POINT_CLEANUP_RATE, Double.class);
+ String consistentHashAlgorithm = MapUtil.getWithDefault(map, PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM,
+ null, String.class);
+
+ Integer numProbes = MapUtil.getWithDefault(map, PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES,
+ DEFAULT_NUM_PROBES);
+
return new DegraderLoadBalancerStrategyConfig(
updateIntervalMs, updateOnlyAtInterval, pointsPerWeight, hashMethod, hashConfig,
clock, initialRecoveryLevel, ringRampFactor, highWaterMark, lowWaterMark,
globalStepUp, globalStepDown, minClusterCallCountHighWaterMark,
- minClusterCallCountLowWaterMark, hashRingPointCleanUpRate);
+ minClusterCallCountLowWaterMark, hashRingPointCleanUpRate,
+ consistentHashAlgorithm, numProbes);
}
/**
@@ -302,6 +322,16 @@ public double getHashRingPointCleanUpRate()
return _hashRingPointCleanUpRate;
}
+ public String getConsistentHashAlgorithm()
+ {
+ return _consistentHashAlgorithm;
+ }
+
+ public int getNumProbes()
+ {
+ return _numProbes;
+ }
+
@Override
public String toString()
{
Oops, something went wrong.

0 comments on commit 319b651

Please sign in to comment.