Skip to content
Permalink
Browse files
HBASE-22349 slop in StochasticLoadBalancer (#4371)
Signed-off-by: Andrew Purtell <apurtell@apache.org>
  • Loading branch information
d-c-manning committed Apr 28, 2022
1 parent ec78c26 commit f848c61e3d27017d78d6244bdb4c95373ba451c6
Showing 9 changed files with 254 additions and 136 deletions.
@@ -25,6 +25,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
@@ -139,6 +140,28 @@ protected final boolean idleRegionServerExist(BalancerClusterState c){
return isServerExistsWithMoreRegions && isServerExistsWithZeroRegions;
}

protected final boolean sloppyRegionServerExist(ClusterLoadState cs) {
if (slop < 0) {
LOG.debug("Slop is less than zero, not checking for sloppiness.");
return false;
}
float average = cs.getLoadAverage(); // for logging
int floor = (int) Math.floor(average * (1 - slop));
int ceiling = (int) Math.ceil(average * (1 + slop));
if (!(cs.getMaxLoad() > ceiling || cs.getMinLoad() < floor)) {
NavigableMap<ServerAndLoad, List<RegionInfo>> serversByLoad = cs.getServersByLoad();
if (LOG.isTraceEnabled()) {
// If nothing to balance, then don't say anything unless trace-level logging.
LOG.trace("Skipping load balancing because balanced cluster; " + "servers=" +
cs.getNumServers() + " regions=" + cs.getNumRegions() + " average=" + average +
" mostloaded=" + serversByLoad.lastKey().getLoad() + " leastloaded=" +
serversByLoad.firstKey().getLoad());
}
return false;
}
return true;
}

/**
* Generates a bulk assignment plan to be used on cluster startup using a
* simple round-robin assignment.
@@ -372,16 +395,6 @@ public Map<ServerName, List<RegionInfo>> retainAssignment(Map<RegionInfo, Server
return Collections.unmodifiableMap(assignments);
}

protected final float normalizeSlop(float slop) {
if (slop < 0) {
return 0;
}
if (slop > 1) {
return 1;
}
return slop;
}

protected float getDefaultSlop() {
return 0.2f;
}
@@ -394,7 +407,7 @@ private RegionHDFSBlockLocationFinder createRegionLocationFinder(Configuration c
}

protected void loadConf(Configuration conf) {
this.slop = normalizeSlop(conf.getFloat("hbase.regions.slop", getDefaultSlop()));
this.slop = conf.getFloat("hbase.regions.slop", getDefaultSlop());
this.rackManager = new RackManager(conf);
useRegionFinder = conf.getBoolean("hbase.master.balancer.uselocality", true);
if (useRegionFinder) {
@@ -191,24 +191,9 @@ private boolean needsBalance(BalancerClusterState c) {
if (idleRegionServerExist(c)) {
return true;
}

// Check if we even need to do any load balancing
// HBASE-3681 check sloppiness first
float average = cs.getLoadAverage(); // for logging
int floor = (int) Math.floor(average * (1 - slop));
int ceiling = (int) Math.ceil(average * (1 + slop));
if (!(cs.getMaxLoad() > ceiling || cs.getMinLoad() < floor)) {
NavigableMap<ServerAndLoad, List<RegionInfo>> serversByLoad = cs.getServersByLoad();
if (LOG.isTraceEnabled()) {
// If nothing to balance, then don't say anything unless trace-level logging.
LOG.trace("Skipping load balancing because balanced cluster; " + "servers=" +
cs.getNumServers() + " regions=" + cs.getNumRegions() + " average=" + average +
" mostloaded=" + serversByLoad.lastKey().getLoad() + " leastloaded=" +
serversByLoad.firstKey().getLoad());
}
return false;
}
return true;
return sloppyRegionServerExist(cs);
}

/**
@@ -203,11 +203,6 @@ List<CandidateGenerator> getCandidateGenerators() {
return this.candidateGenerators;
}

@Override
protected float getDefaultSlop() {
return 0.001f;
}

protected List<CandidateGenerator> createCandidateGenerators() {
List<CandidateGenerator> candidateGenerators = new ArrayList<CandidateGenerator>(4);
candidateGenerators.add(GeneratorType.RANDOM.ordinal(), new RandomCandidateGenerator());
@@ -357,6 +352,12 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) {
return true;
}

if (sloppyRegionServerExist(cs)) {
LOG.info("Running balancer because cluster has sloppy server(s)."+
" function cost={}", functionCost());
return true;
}

double total = 0.0;
for (CostFunction c : costFunctions) {
if (!c.isNeeded()) {
@@ -60,83 +60,122 @@ public class BalancerTestBase {
protected static Configuration conf;

protected int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 };
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 };

// int[testnum][servernumber] -> numregions
protected int[][] clusterStateMocks = new int[][]{
// 1 node
new int[]{0},
new int[]{1},
new int[]{10},
// 2 node
new int[]{0, 0},
new int[]{2, 0},
new int[]{2, 1},
new int[]{2, 2},
new int[]{2, 3},
new int[]{2, 4},
new int[]{1, 1},
new int[]{0, 1},
new int[]{10, 1},
new int[]{514, 1432},
new int[]{48, 53},
// 3 node
new int[]{0, 1, 2},
new int[]{1, 2, 3},
new int[]{0, 2, 2},
new int[]{0, 3, 0},
new int[]{0, 4, 0},
new int[]{20, 20, 0},
// 4 node
new int[]{0, 1, 2, 3},
new int[]{4, 0, 0, 0},
new int[]{5, 0, 0, 0},
new int[]{6, 6, 0, 0},
new int[]{6, 2, 0, 0},
new int[]{6, 1, 0, 0},
new int[]{6, 0, 0, 0},
new int[]{4, 4, 4, 7},
new int[]{4, 4, 4, 8},
new int[]{0, 0, 0, 7},
// 5 node
new int[]{1, 1, 1, 1, 4},
// 6 nodes
new int[]{1500, 500, 500, 500, 10, 0},
new int[]{1500, 500, 500, 500, 500, 0},
// more nodes
new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
new int[]{10, 7, 12, 8, 11, 10, 9, 14},
new int[]{13, 14, 6, 10, 10, 10, 8, 10},
new int[]{130, 14, 60, 10, 100, 10, 80, 10},
new int[]{130, 140, 60, 100, 100, 100, 80, 100},
new int[]{0, 5 , 5, 5, 5},
largeCluster,
// 1 node
new int[]{0},
new int[]{1},
new int[]{10},
// 2 node
new int[]{0, 0},
new int[]{2, 0},
new int[]{2, 1},
new int[]{2, 2},
new int[]{2, 3},
new int[]{2, 4},
new int[]{1, 1},
new int[]{0, 1},
new int[]{10, 1},
new int[]{514, 1432},
new int[]{48, 53},
// 3 node
new int[]{0, 1, 2},
new int[]{1, 2, 3},
new int[]{0, 2, 2},
new int[]{0, 3, 0},
new int[]{0, 4, 0},
new int[]{20, 20, 0},
// 4 node
new int[]{0, 1, 2, 3},
new int[]{4, 0, 0, 0},
new int[]{5, 0, 0, 0},
new int[]{6, 6, 0, 0},
new int[]{6, 2, 0, 0},
new int[]{6, 1, 0, 0},
new int[]{6, 0, 0, 0},
new int[]{4, 4, 4, 7},
new int[]{4, 4, 4, 8},
new int[]{0, 0, 0, 7},
// 5 node
new int[]{1, 1, 1, 1, 4},
// 6 nodes
new int[]{1500, 500, 500, 500, 10, 0},
new int[]{1500, 500, 500, 500, 500, 0},
// more nodes
new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
new int[]{10, 7, 12, 8, 11, 10, 9, 14},
new int[]{13, 14, 6, 10, 10, 10, 8, 10},
new int[]{130, 14, 60, 10, 100, 10, 80, 10},
new int[]{130, 140, 60, 100, 100, 100, 80, 100},
new int[]{0, 5 , 5, 5, 5},
largeCluster,

};

// int[testnum][servernumber] -> numregions
protected int[][] clusterStateMocksWithNoSlop = new int[][] {
// 1 node
new int[]{0},
new int[]{1},
new int[]{10},
// 2 node
new int[]{0, 0},
new int[]{2, 1},
new int[]{2, 2},
new int[]{2, 3},
new int[]{1, 1},
new int[]{80, 120},
new int[]{1428, 1432},
// more nodes
new int[]{100, 90, 120, 90, 110, 100, 90, 120},
};

// int[testnum][servernumber] -> numregions
protected int[][] clusterStateMocksWithSlop = new int[][] {
// 2 node
new int[]{1, 4},
new int[]{10, 20},
new int[]{80, 123},
// more nodes
new int[]{100, 100, 100, 100, 100, 100, 100, 100, 100, 200},
new int[] {
10, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
},
};

// This class is introduced because IP to rack resolution can be lengthy.
public static class MockMapping implements DNSToSwitchMapping {
@@ -351,6 +390,23 @@ protected void updateLoad(final Map<ServerName, ServerAndLoad> map,
map.put(sn, sal);
}

protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[][] mockCluster) {
// dimension1: table, dimension2: regions per server
int numTables = mockCluster.length;
TreeMap<ServerName, List<RegionInfo>> servers = new TreeMap<>();
for (int i = 0; i < numTables; i++) {
TableName tableName = TableName.valueOf("table" + i);
for (int j = 0; j < mockCluster[i].length; j++) {
ServerName serverName = ServerName.valueOf("server" + j, 1000, -1);
int numRegions = mockCluster[i][j];
List<RegionInfo> regions = createRegions(numRegions, tableName);
servers.putIfAbsent(serverName, new ArrayList<>());
servers.get(serverName).addAll(regions);
}
}
return servers;
}

protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[] mockCluster) {
return mockClusterServers(mockCluster, -1);
}
@@ -46,7 +46,6 @@ public class StochasticBalancerTestBase extends BalancerTestBase {
public static void beforeAllTests() throws Exception {
conf = HBaseConfiguration.create();
conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
conf.setFloat("hbase.regions.slop", 0.0f);
conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true);
loadBalancer = new StochasticLoadBalancer(dummyMetricsStochasticBalancer);
@@ -165,28 +165,19 @@ private void testImpactOfBalanceClusterOverall(boolean useLoadOfAllTable) throws
}

@Test
public void testBalanceClusterOverallStrictly() throws Exception {
int[] regionNumOfTable1PerServer = { 3, 3, 4, 4, 4, 4, 5, 5, 5 };
int[] regionNumOfTable2PerServer = { 2, 2, 2, 2, 2, 2, 2, 2, 1 };
TreeMap<ServerName, List<RegionInfo>> serverRegionInfo = new TreeMap<>();
List<ServerAndLoad> serverAndLoads = new ArrayList<>();
for (int i = 0; i < regionNumOfTable1PerServer.length; i++) {
ServerName serverName = ServerName.valueOf("server" + i, 1000, -1);
List<RegionInfo> regions1 =
createRegions(regionNumOfTable1PerServer[i], TableName.valueOf("table1"));
List<RegionInfo> regions2 =
createRegions(regionNumOfTable2PerServer[i], TableName.valueOf("table2"));
regions1.addAll(regions2);
serverRegionInfo.put(serverName, regions1);
ServerAndLoad serverAndLoad = new ServerAndLoad(serverName,
regionNumOfTable1PerServer[i] + regionNumOfTable2PerServer[i]);
serverAndLoads.add(serverAndLoad);
}
HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> LoadOfAllTable =
public void testBalanceClusterOverallStrictly() {
int[][] regionsPerServerPerTable = new int[][]{
new int[]{ 3, 3, 4, 4, 4, 4, 5, 5, 5 },
new int[]{ 2, 2, 2, 2, 2, 2, 2, 2, 1 },
};
TreeMap<ServerName, List<RegionInfo>> serverRegionInfo =
mockClusterServers(regionsPerServerPerTable);
List<ServerAndLoad> serverAndLoads = convertToList(serverRegionInfo);
Map<TableName, TreeMap<ServerName, List<RegionInfo>>> loadOfAllTable =
mockClusterServersWithTables(serverRegionInfo);
loadBalancer.setClusterLoad((Map) LoadOfAllTable);
List<RegionPlan> partialplans = loadBalancer.balanceTable(TableName.valueOf("table1"),
LoadOfAllTable.get(TableName.valueOf("table1")));
loadBalancer.setClusterLoad((Map) loadOfAllTable);
List<RegionPlan> partialplans = loadBalancer.balanceTable(TableName.valueOf("table0"),
loadOfAllTable.get(TableName.valueOf("table0")));
List<ServerAndLoad> balancedServerLoads =
reconcile(serverAndLoads, partialplans, serverRegionInfo);
for (ServerAndLoad serverAndLoad : balancedServerLoads) {

0 comments on commit f848c61

Please sign in to comment.