From d4a53417fa7762b5caf15aad2a0147a46df7f8ee Mon Sep 17 00:00:00 2001 From: ddupg Date: Mon, 25 May 2020 10:50:18 +0800 Subject: [PATCH] HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count --- .../normalizer/AbstractRegionNormalizer.java | 6 +++ .../master/normalizer/MergeNormalizer.java | 11 +++-- .../normalizer/SimpleRegionNormalizer.java | 29 ++++++++----- .../TestSimpleRegionNormalizer.java | 43 +++++++++++++++++++ 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java index 8b17f77dc914..ef4b004768e4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; + import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Size; @@ -42,6 +43,11 @@ @InterfaceAudience.Private public abstract class AbstractRegionNormalizer implements RegionNormalizer { private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class); + + public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY = + "hbase.normalizer.min.region.count"; + public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3; + protected MasterServices masterServices; protected MasterRpcServices masterRpcServices; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java index 444c27c28681..963e7445a680 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java @@ -23,10 +23,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,9 +67,12 @@ public class MergeNormalizer extends AbstractRegionNormalizer { private int minRegionAge; private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; - public MergeNormalizer() { - Configuration conf = HBaseConfiguration.create(); - minRegionCount = conf.getInt("hbase.normalizer.min.region.count", 3); + @Override + public void setMasterServices(MasterServices masterServices) { + super.setMasterServices(masterServices); + Configuration conf = masterServices.getConfiguration(); + minRegionCount = conf.getInt(HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, + HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT); minRegionAge = conf.getInt("hbase.normalizer.min.region.merge.age", 3); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index bd90f5b76c33..30d8284bbc55 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -22,10 +22,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import org.apache.hadoop.hbase.HBaseConfiguration; + import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -54,8 +55,12 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer { private int minRegionCount; private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; - public SimpleRegionNormalizer() { - minRegionCount = HBaseConfiguration.create().getInt("hbase.normalizer.min.region.count", 3); + @Override + public void setMasterServices(MasterServices masterServices) { + super.setMasterServices(masterServices); + minRegionCount = masterServices.getConfiguration().getInt( + HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, + HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT); } @Override @@ -112,11 +117,7 @@ public List computePlanForTable(TableName table) throws HBase List tableRegions = masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table); - if (tableRegions == null || tableRegions.size() < minRegionCount) { - int nrRegions = tableRegions == null ? 0 : tableRegions.size(); - LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is " - + "{}, not running normalizer", - table, nrRegions, minRegionCount); + if (tableRegions == null) { return null; } @@ -131,9 +132,15 @@ public List computePlanForTable(TableName table) throws HBase } if (mergeEnabled) { - List mergePlans = getMergeNormalizationPlan(table); - if (mergePlans != null) { - plans.addAll(mergePlans); + if (tableRegions.size() < minRegionCount) { + LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is " + + "{}, not running normalizer", + table, tableRegions.size(), minRegionCount); + } else { + List mergePlans = getMergeNormalizationPlan(table); + if (mergePlans != null) { + plans.addAll(mergePlans); + } } } if (plans.isEmpty()) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index a05b3a8c4911..5c1c4e09cc04 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -19,6 +19,7 @@ import static java.lang.String.format; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -30,7 +31,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ServerName; @@ -526,6 +530,45 @@ public void testSplitWithTargetRegionSize() throws Exception { assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion()); } + @Test + public void testSplitIfTooFewRegions() throws HBaseIOException { + final TableName tableName = TableName.valueOf(name.getMethodName()); + List RegionInfo = new ArrayList<>(); + Map regionSizes = new HashMap<>(); + + RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("aaa")) + .setEndKey(Bytes.toBytes("bbb")) + .build(); + RegionInfo.add(hri1); + regionSizes.put(hri1.getRegionName(), 1); + + RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("bbb")) + .setEndKey(Bytes.toBytes("ccc")) + .build(); + RegionInfo.add(hri2); + regionSizes.put(hri2.getRegionName(), 1); + // the third region is huge one + RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("ccc")) + .setEndKey(Bytes.toBytes("ddd")) + .build(); + RegionInfo.add(hri3); + regionSizes.put(hri3.getRegionName(), 10); + + setupMocksForNormalizer(regionSizes, RegionInfo); + + Configuration configuration = HBaseConfiguration.create(); + configuration.setInt(AbstractRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4); + when(masterServices.getConfiguration()).thenReturn(configuration); + + List plans = normalizer.computePlanForTable(tableName); + assertNotNull(plans); + NormalizationPlan plan = plans.get(0); + assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo()); + } + @SuppressWarnings("MockitoCast") private void setupMocksForNormalizer(Map regionSizes, List RegionInfo) {