Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HBASE-24416 RegionNormalizer spliting region should not be limited by… #1788

Merged
merged 1 commit into from
May 28, 2020
Merged

HBASE-24416 RegionNormalizer spliting region should not be limited by… #1788

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions ...erver/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,12 @@
public class SimpleRegionNormalizer implements RegionNormalizer {

private static final Logger LOG = LoggerFactory.getLogger(SimpleRegionNormalizer.class);
private static final int MIN_REGION_COUNT = 3;

public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY =
"hbase.normalizer.min.region.count";
public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3;

protected int minRegionCount;
private MasterServices masterServices;
private MasterRpcServices masterRpcServices;
private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
Expand All @@ -75,6 +80,9 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
@Override
public void setMasterServices(MasterServices masterServices) {
this.masterServices = masterServices;
minRegionCount = masterServices.getConfiguration().getInt(
HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY,
HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT);
}

@Override
Expand Down Expand Up @@ -148,14 +156,17 @@ public List<NormalizationPlan> computePlanForTable(TableName table) throws HBase
List<RegionInfo> tableRegions = masterServices.getAssignmentManager().getRegionStates().
getRegionsOfTable(table);

//TODO: should we make min number of regions a config param?
if (tableRegions == null || tableRegions.size() < MIN_REGION_COUNT) {
int nrRegions = tableRegions == null ? 0 : tableRegions.size();
LOG.debug("Table " + table + " has " + nrRegions + " regions, required min number"
+ " of regions for normalizer to run is " + MIN_REGION_COUNT + ", not running normalizer");
if (tableRegions == null) {
return null;
}

if (tableRegions.size() < minRegionCount) {
LOG.debug("Table " + table + " has " + tableRegions.size() + " regions, required min number"
+ " of regions for normalizer merging to run is " + minRegionCount
+ ", not running normalizer merging");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good to use parameters with {} for variables.

mergeEnabled = false;
}

LOG.debug("Computing normalization plan for table: " + table +
", number of regions: " + tableRegions.size());

Expand Down
45 changes: 45 additions & 0 deletions ...r/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.master.normalizer;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
Expand All @@ -28,7 +29,10 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.RegionMetrics;
import org.apache.hadoop.hbase.ServerName;
Expand Down Expand Up @@ -476,6 +480,47 @@ public void testSplitWithTargetRegionSize() throws Exception {
assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion());
}

@Test
public void testSplitIfTooFewRegions() throws HBaseIOException {
final TableName tableName = TableName.valueOf(name.getMethodName());
List<RegionInfo> RegionInfo = new ArrayList<>();
Map<byte[], Integer> regionSizes = new HashMap<>();

RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
.setStartKey(Bytes.toBytes("aaa"))
.setEndKey(Bytes.toBytes("bbb"))
.build();
RegionInfo.add(hri1);
regionSizes.put(hri1.getRegionName(), 1);

RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
.setStartKey(Bytes.toBytes("bbb"))
.setEndKey(Bytes.toBytes("ccc"))
.build();
RegionInfo.add(hri2);
regionSizes.put(hri2.getRegionName(), 1);
// the third region is huge one
RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
.setStartKey(Bytes.toBytes("ccc"))
.setEndKey(Bytes.toBytes("ddd"))
.build();
RegionInfo.add(hri3);
regionSizes.put(hri3.getRegionName(), 10);

setupMocksForNormalizer(regionSizes, RegionInfo);

Configuration configuration = HBaseConfiguration.create();
configuration.setInt(SimpleRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4);
when(masterServices.getConfiguration()).thenReturn(configuration);
normalizer.setMasterServices(masterServices);

List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
assertNotNull(plans);
assertEquals(1, plans.size());
NormalizationPlan plan = plans.get(0);
assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo());
}

@SuppressWarnings("MockitoCast")
protected void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
List<RegionInfo> RegionInfo) {
Expand Down