From 780dca9c7232942174f6f1ccc1cb479544a20a68 Mon Sep 17 00:00:00 2001 From: XinSun Date: Tue, 26 May 2020 17:03:14 +0800 Subject: [PATCH] HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770) Signed-off-by: Guanghao Zhang --- .../normalizer/AbstractRegionNormalizer.java | 10 +++++ .../master/normalizer/MergeNormalizer.java | 10 ++--- .../normalizer/SimpleRegionNormalizer.java | 25 +++++------ .../TestSimpleRegionNormalizer.java | 43 +++++++++++++++++++ 4 files changed, 69 insertions(+), 19 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java index 8b17f77dc914..c79c1d440578 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; + import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Size; @@ -42,8 +43,14 @@ @InterfaceAudience.Private public abstract class AbstractRegionNormalizer implements RegionNormalizer { private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class); + + public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY = + "hbase.normalizer.min.region.count"; + public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3; + protected MasterServices masterServices; protected MasterRpcServices masterRpcServices; + protected int minRegionCount; /** * Set the master service. @@ -52,6 +59,9 @@ public abstract class AbstractRegionNormalizer implements RegionNormalizer { @Override public void setMasterServices(MasterServices masterServices) { this.masterServices = masterServices; + minRegionCount = masterServices.getConfiguration().getInt( + HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, + HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java index 444c27c28681..f006103f556b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java @@ -23,10 +23,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,13 +63,13 @@ public class MergeNormalizer extends AbstractRegionNormalizer { private static final Logger LOG = LoggerFactory.getLogger(MergeNormalizer.class); - private int minRegionCount; private int minRegionAge; private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; - public MergeNormalizer() { - Configuration conf = HBaseConfiguration.create(); - minRegionCount = conf.getInt("hbase.normalizer.min.region.count", 3); + @Override + public void setMasterServices(MasterServices masterServices) { + super.setMasterServices(masterServices); + Configuration conf = masterServices.getConfiguration(); minRegionAge = conf.getInt("hbase.normalizer.min.region.merge.age", 3); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index bd90f5b76c33..b68c0682f1e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -22,7 +22,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import org.apache.hadoop.hbase.HBaseConfiguration; + import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -51,13 +51,8 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer { private static final Logger LOG = LoggerFactory.getLogger(SimpleRegionNormalizer.class); - private int minRegionCount; private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; - public SimpleRegionNormalizer() { - minRegionCount = HBaseConfiguration.create().getInt("hbase.normalizer.min.region.count", 3); - } - @Override public void planSkipped(RegionInfo hri, PlanType type) { skippedCount[type.ordinal()]++; @@ -112,11 +107,7 @@ public List computePlanForTable(TableName table) throws HBase List tableRegions = masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table); - if (tableRegions == null || tableRegions.size() < minRegionCount) { - int nrRegions = tableRegions == null ? 0 : tableRegions.size(); - LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is " - + "{}, not running normalizer", - table, nrRegions, minRegionCount); + if (tableRegions == null) { return null; } @@ -131,9 +122,15 @@ public List computePlanForTable(TableName table) throws HBase } if (mergeEnabled) { - List mergePlans = getMergeNormalizationPlan(table); - if (mergePlans != null) { - plans.addAll(mergePlans); + if (tableRegions.size() < minRegionCount) { + LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run" + + " is {}, not running normalizer", + table, tableRegions.size(), minRegionCount); + } else { + List mergePlans = getMergeNormalizationPlan(table); + if (mergePlans != null) { + plans.addAll(mergePlans); + } } } if (plans.isEmpty()) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index a05b3a8c4911..5c1c4e09cc04 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -19,6 +19,7 @@ import static java.lang.String.format; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -30,7 +31,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ServerName; @@ -526,6 +530,45 @@ public void testSplitWithTargetRegionSize() throws Exception { assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion()); } + @Test + public void testSplitIfTooFewRegions() throws HBaseIOException { + final TableName tableName = TableName.valueOf(name.getMethodName()); + List RegionInfo = new ArrayList<>(); + Map regionSizes = new HashMap<>(); + + RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("aaa")) + .setEndKey(Bytes.toBytes("bbb")) + .build(); + RegionInfo.add(hri1); + regionSizes.put(hri1.getRegionName(), 1); + + RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("bbb")) + .setEndKey(Bytes.toBytes("ccc")) + .build(); + RegionInfo.add(hri2); + regionSizes.put(hri2.getRegionName(), 1); + // the third region is huge one + RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("ccc")) + .setEndKey(Bytes.toBytes("ddd")) + .build(); + RegionInfo.add(hri3); + regionSizes.put(hri3.getRegionName(), 10); + + setupMocksForNormalizer(regionSizes, RegionInfo); + + Configuration configuration = HBaseConfiguration.create(); + configuration.setInt(AbstractRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4); + when(masterServices.getConfiguration()).thenReturn(configuration); + + List plans = normalizer.computePlanForTable(tableName); + assertNotNull(plans); + NormalizationPlan plan = plans.get(0); + assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo()); + } + @SuppressWarnings("MockitoCast") private void setupMocksForNormalizer(Map regionSizes, List RegionInfo) {