Permalink
Browse files

Add new consistent hash ring implementation which is more balanced an…

…d uses less memory

RB=772345
G=si-core-reviewers
R=dhoa,cxu
A=dhoa
  • Loading branch information...
angxu committed Jul 25, 2016
1 parent 3578bf5 commit 319b651a4f2a4d763ec0085019cf4af9a15db55b
Showing with 1,010 additions and 363 deletions.
  1. +3 −0 build.gradle
  2. +15 −0 d2-benchmark/build.gradle
  3. +132 −0 d2-benchmark/src/jmh/java/com/linkedin/d2/util/hashing/ConsistentHashRingBenchmark.java
  4. +23 −0 d2-schemas/src/main/pegasus/com/linkedin/d2/D2LoadBalancerStrategyProperties.pdsc
  5. +1 −0 d2/build.gradle
  6. +34 −1 d2/src/main/java/com/linkedin/d2/balancer/config/LoadBalancerStrategyPropertiesConverter.java
  7. +2 −0 d2/src/main/java/com/linkedin/d2/balancer/properties/PropertyKeys.java
  8. +34 −4 ...rc/main/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerStrategyConfig.java
  9. +17 −154 d2/src/main/java/com/linkedin/d2/balancer/strategies/degrader/DegraderRingFactory.java
  10. +43 −0 d2/src/main/java/com/linkedin/d2/balancer/strategies/degrader/MPConsistentHashRingFactory.java
  11. +195 −0 ...c/main/java/com/linkedin/d2/balancer/strategies/degrader/PointBasedConsistentHashRingFactory.java
  12. +198 −0 d2/src/main/java/com/linkedin/d2/balancer/util/hashing/MPConsistentHashRing.java
  13. +9 −1 d2/src/test/java/com/linkedin/d2/balancer/config/LoadBalancerStrategyPropertiesConverterTest.java
  14. +1 −1 d2/src/test/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerStateTest.java
  15. +9 −0 ...est/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerStrategyConfigTest.java
  16. +42 −141 d2/src/test/java/com/linkedin/d2/balancer/strategies/degrader/DegraderLoadBalancerTest.java
  17. +3 −4 d2/src/test/java/com/linkedin/d2/balancer/strategies/degrader/DegraderRingFactoryTest.java
  18. +71 −51 d2/src/test/java/com/linkedin/d2/balancer/util/hashing/ConsistentHashKeyMapperTest.java
  19. +165 −0 d2/src/test/java/com/linkedin/d2/balancer/util/hashing/MPConsistentHashTest.java
  20. +12 −6 restli-int-test/src/test/java/com/linkedin/restli/examples/TestAllPartitionsRequestBuilder.java
  21. +1 −0 settings.gradle
View
@@ -51,6 +51,8 @@ project.ext.externalDependency = [
'javaxInject': 'javax.inject:javax.inject:1',
'jdkTools': files("${System.getProperty('java.home')}/../lib/tools.jar"),
'jetty': 'org.eclipse.jetty.aggregate:jetty-all:8.1.8.v20121106',
"jmhCore": "org.openjdk.jmh:jmh-core:1.11",
"jmhAnnotations": "org.openjdk.jmh:jmh-generator-annprocess:1.11",
'log4j2Api': 'org.apache.logging.log4j:log4j-api:2.0.2',
'log4j2Core': 'org.apache.logging.log4j:log4j-core:2.0.2',
'log4jLog4j2': 'org.apache.logging.log4j:log4j-1.2-api:2.0.2',
@@ -66,6 +68,7 @@ project.ext.externalDependency = [
'snappy': 'org.iq80.snappy:snappy:0.3',
'testng': 'org.testng:testng:6.9.9',
'velocity': 'org.apache.velocity:velocity:1.5',
'zero_allocation_hashing': 'net.openhft:zero-allocation-hashing:0.4',
'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.6',
// for restli-spring-bridge ONLY, we must keep these dependencies isolated
View
@@ -0,0 +1,15 @@
plugins {
id 'me.champeau.gradle.jmh' version '0.3.0'
}
jmh {
include = '.*ConsistentHashRingBenchmark.*'
zip64 = true
}
dependencies {
jmh project(':d2')
jmh externalDependency.jmhCore
jmh externalDependency.jmhAnnotations
}
@@ -0,0 +1,132 @@
/*
Copyright (c) 2016 LinkedIn Corp.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package com.linkedin.d2.util.hashing;
import com.linkedin.d2.balancer.util.hashing.ConsistentHashRing;
import com.linkedin.d2.balancer.util.hashing.MPConsistentHashRing;
import com.linkedin.d2.balancer.util.hashing.Ring;
import java.net.URI;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
/**
* @author Ang Xu
*/
@Fork(2)
@Warmup(iterations = 5)
@Measurement(iterations = 5)
public class ConsistentHashRingBenchmark {
@State(Scope.Benchmark)
public static class MPCHash_10Hosts_11Probes_State {
Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(10, 100), 11);
Random _random = new Random();
}
@Benchmark
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public URI measureMPCHash_10Hosts_11Probes(MPCHash_10Hosts_11Probes_State state) {
return state._ring.get(state._random.nextInt());
}
@State(Scope.Benchmark)
public static class MPCHash_10Hosts_21Probes_State {
Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(10, 100), 21);
Random _random = new Random();
}
@Benchmark
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public URI measureMPCHash_10Hosts_21Probes(MPCHash_10Hosts_21Probes_State state) {
return state._ring.get(state._random.nextInt());
}
@State(Scope.Benchmark)
public static class MPCHash_100Hosts_11Probes_State {
Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(100, 100), 11);
Random _random = new Random();
}
@Benchmark
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public URI measureMPCHash_100Hosts_11Probes(MPCHash_100Hosts_11Probes_State state) {
return state._ring.get(state._random.nextInt());
}
@State(Scope.Benchmark)
public static class MPCHash_100Hosts_21Probes_State {
Ring<URI> _ring = new MPConsistentHashRing<>(buildPointsMap(100, 100), 21);
Random _random = new Random();
}
@Benchmark
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public URI measureMPCHash_100Hosts_21Probes(MPCHash_100Hosts_21Probes_State state) {
return state._ring.get(state._random.nextInt());
}
@State(Scope.Benchmark)
public static class ConsistentHashRing_10Hosts_100PointsPerHost_State {
Ring<URI> _ring = new ConsistentHashRing<URI>(buildPointsMap(10, 100));
Random _random = new Random();
}
@Benchmark
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public URI measureonsistentHashRing_10Hosts_100PointsPerHost(ConsistentHashRing_10Hosts_100PointsPerHost_State state) {
return state._ring.get(state._random.nextInt());
}
@State(Scope.Benchmark)
public static class ConsistentHashRing_100Hosts_100PointsPerHost_State {
Ring<URI> _ring = new ConsistentHashRing<URI>(buildPointsMap(100, 100));
Random _random = new Random();
}
@Benchmark
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public URI measureonsistentHashRing_100Hosts_100PointsPerHost(ConsistentHashRing_100Hosts_100PointsPerHost_State state) {
return state._ring.get(state._random.nextInt());
}
private static Map<URI, Integer> buildPointsMap(int numHosts, int numPointsPerHost) {
return IntStream.range(0, numHosts).boxed().collect(
Collectors.toMap(
key -> URI.create(String.format("app-%04d.linkedin.com", key)),
value -> numPointsPerHost));
}
}
@@ -118,6 +118,29 @@
"type": "double",
"doc": "The highest ratio of unused entries over the total entries of the Ring points that d2 maintains.",
"optional": true
},
{
"name": "consistentHashAlgorithm",
"type": {
"name": "ConsistentHashAlgorithmEnum",
"type": "enum",
"symbols": [
"POINT_BASED",
"MULTI_PROBE"
],
"symbolDocs": {
"POINT_BASED": "Point-based consistent hash ring. The more points the ring has, the more balanced it is.",
"MULTI_PROBE": "Multi-probe consistent hash. The more probes to use, the more balanced the ring is."
}
},
"doc": "Consistent hash algorithm the d2 load balancer should use. Defaults to POINT_BASED.",
"optional": true
},
{
"name": "numberOfProbes",
"type": "int",
"doc": "The number of probes used to look up a key in consistent hash ring. Defaults to 21.",
"optional": true
}
]
}
View
@@ -16,6 +16,7 @@ dependencies {
compile externalDependency.jacksonCore
compile externalDependency.jacksonDataBind
compile externalDependency.jdkTools
compile externalDependency.zero_allocation_hashing
testCompile externalDependency.testng
testCompile externalDependency.commonsIo
testCompile externalDependency.easymock
@@ -16,14 +16,15 @@
package com.linkedin.d2.balancer.config;
import com.linkedin.d2.ConsistentHashAlgorithmEnum;
import com.linkedin.d2.D2LoadBalancerStrategyProperties;
import com.linkedin.d2.balancer.properties.PropertyKeys;
import com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3;
import com.linkedin.d2.balancer.strategies.degrader.DegraderRingFactory;
import com.linkedin.d2.balancer.util.hashing.URIRegexHash;
import com.linkedin.d2.hashConfigType;
import com.linkedin.d2.hashMethodEnum;
import com.linkedin.data.template.StringArray;
import com.linkedin.data.template.StringMap;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -126,6 +127,22 @@
{
map.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_ONLY_AT_INTERVAL, config.isUpdateOnlyAtInterval().toString());
}
if (config.hasConsistentHashAlgorithm())
{
switch (config.getConsistentHashAlgorithm())
{
case MULTI_PROBE:
map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DegraderRingFactory.MULTI_PROBE_CONSISTENT_HASH);
break;
case POINT_BASED:
map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DegraderRingFactory.POINT_BASED_CONSISTENT_HASH);
break;
}
}
if (config.hasNumberOfProbes())
{
map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES, config.getNumberOfProbes().toString());
}
return map;
}
@@ -218,6 +235,22 @@ else if (DegraderLoadBalancerStrategyV3.HASH_METHOD_URI_REGEX.equalsIgnoreCase(h
coerce(properties.get(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_ONLY_AT_INTERVAL),
Boolean.class));
}
if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM))
{
String consistentHashAlgorithm = coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM), String.class);
if (DegraderRingFactory.POINT_BASED_CONSISTENT_HASH.equalsIgnoreCase(consistentHashAlgorithm))
{
config.setConsistentHashAlgorithm(ConsistentHashAlgorithmEnum.POINT_BASED);
}
else if (DegraderRingFactory.MULTI_PROBE_CONSISTENT_HASH.equalsIgnoreCase(consistentHashAlgorithm))
{
config.setConsistentHashAlgorithm(ConsistentHashAlgorithmEnum.MULTI_PROBE);
}
}
if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES))
{
config.setNumberOfProbes(coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES), Integer.class));
}
return config;
}
}
@@ -84,6 +84,8 @@
public static final String HTTP_LB_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK = "http.loadBalancer.clusterMinCallCount.highWaterMark";
public static final String HTTP_LB_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK = "http.loadBalancer.clusterMinCallCount.lowWaterMark";
public static final String HTTP_LB_HASHRING_POINT_CLEANUP_RATE = "http.loadBalancer.hashRingPointCleanupRate";
public static final String HTTP_LB_CONSISTENT_HASH_ALGORITHM = "http.loadBalancer.consistentHashAlgorithm";
public static final String HTTP_LB_CONSISTENT_HASH_NUM_PROBES = "http.loadBalancer.consistentHashNumProbes";
//used by service metadata properties
public static final String SERVICE_FOLLOW_REDIRECTION_MAX_HOP = "followRedirection.maxHop";
@@ -17,6 +17,7 @@
package com.linkedin.d2.balancer.strategies.degrader;
import com.linkedin.d2.balancer.properties.PropertyKeys;
import com.linkedin.d2.balancer.util.hashing.MPConsistentHashRing;
import java.util.Collections;
import java.util.Map;
@@ -64,6 +65,9 @@
private final double _hashRingPointCleanUpRate;
private final String _consistentHashAlgorithm;
private final int _numProbes;
public static final Clock DEFAULT_CLOCK = SystemClock.instance();
public static final double DEFAULT_INITIAL_RECOVERY_LEVEL = 0.01;
public static final double DEFAULT_RAMP_FACTOR = 1.0;
@@ -84,14 +88,17 @@
public static final double DEFAULT_HASHRING_POINT_CLEANUP_RATE = 0.20;
public static final int DEFAULT_NUM_PROBES = MPConsistentHashRing.DEFAULT_NUM_PROBES;
public DegraderLoadBalancerStrategyConfig(long updateIntervalMs)
{
this(updateIntervalMs, DEFAULT_UPDATE_ONLY_AT_INTERVAL, 100, null, Collections.<String, Object>emptyMap(),
DEFAULT_CLOCK, DEFAULT_INITIAL_RECOVERY_LEVEL, DEFAULT_RAMP_FACTOR, DEFAULT_HIGH_WATER_MARK, DEFAULT_LOW_WATER_MARK,
DEFAULT_GLOBAL_STEP_UP, DEFAULT_GLOBAL_STEP_DOWN,
DEFAULT_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK,
DEFAULT_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK,
DEFAULT_HASHRING_POINT_CLEANUP_RATE);
DEFAULT_HASHRING_POINT_CLEANUP_RATE,
null, DEFAULT_NUM_PROBES);
}
public DegraderLoadBalancerStrategyConfig(DegraderLoadBalancerStrategyConfig config)
@@ -110,7 +117,9 @@ public DegraderLoadBalancerStrategyConfig(DegraderLoadBalancerStrategyConfig con
config.getGlobalStepDown(),
config.getMinClusterCallCountHighWaterMark(),
config.getMinClusterCallCountLowWaterMark(),
config.getHashRingPointCleanUpRate());
config.getHashRingPointCleanUpRate(),
config.getConsistentHashAlgorithm(),
config.getNumProbes());
}
public DegraderLoadBalancerStrategyConfig(long updateIntervalMs,
@@ -127,7 +136,9 @@ public DegraderLoadBalancerStrategyConfig(long updateIntervalMs,
double globalStepDown,
long minCallCountHighWaterMark,
long minCallCountLowWaterMark,
double hashRingPointCleanUpRate)
double hashRingPointCleanUpRate,
String consistentHashAlgorithm,
int numProbes)
{
_updateIntervalMs = updateIntervalMs;
_updateOnlyAtInterval = updateOnlyAtInterval;
@@ -144,6 +155,8 @@ public DegraderLoadBalancerStrategyConfig(long updateIntervalMs,
_minClusterCallCountHighWaterMark = minCallCountHighWaterMark;
_minClusterCallCountLowWaterMark = minCallCountLowWaterMark;
_hashRingPointCleanUpRate = hashRingPointCleanUpRate;
_consistentHashAlgorithm = consistentHashAlgorithm;
_numProbes = numProbes;
}
/**
@@ -214,11 +227,18 @@ public static DegraderLoadBalancerStrategyConfig createHttpConfigFromMap(Map<Str
Double hashRingPointCleanUpRate = MapUtil.getWithDefault(map, PropertyKeys.HTTP_LB_HASHRING_POINT_CLEANUP_RATE,
DEFAULT_HASHRING_POINT_CLEANUP_RATE, Double.class);
String consistentHashAlgorithm = MapUtil.getWithDefault(map, PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM,
null, String.class);
Integer numProbes = MapUtil.getWithDefault(map, PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES,
DEFAULT_NUM_PROBES);
return new DegraderLoadBalancerStrategyConfig(
updateIntervalMs, updateOnlyAtInterval, pointsPerWeight, hashMethod, hashConfig,
clock, initialRecoveryLevel, ringRampFactor, highWaterMark, lowWaterMark,
globalStepUp, globalStepDown, minClusterCallCountHighWaterMark,
minClusterCallCountLowWaterMark, hashRingPointCleanUpRate);
minClusterCallCountLowWaterMark, hashRingPointCleanUpRate,
consistentHashAlgorithm, numProbes);
}
/**
@@ -302,6 +322,16 @@ public double getHashRingPointCleanUpRate()
return _hashRingPointCleanUpRate;
}
public String getConsistentHashAlgorithm()
{
return _consistentHashAlgorithm;
}
public int getNumProbes()
{
return _numProbes;
}
@Override
public String toString()
{
Oops, something went wrong.

0 comments on commit 319b651

Please sign in to comment.