HBASE-24416 RegionNormalizer spliting region should not be limited by…

… hbase.normalizer.min.region.count (#1770) Signed-off-by: Guanghao Zhang <zghao@apache.org>
apache · May 26, 2020 · 780dca9 · 780dca9
1 parent 162e1d0
commit 780dca9
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 19 deletions.
diff --git a/...ver/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java b/...ver/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+
 import org.apache.hadoop.hbase.RegionMetrics;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.Size;
@@ -42,8 +43,14 @@
 @InterfaceAudience.Private
 public abstract class AbstractRegionNormalizer implements RegionNormalizer {
   private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class);
+
+  public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY =
+      "hbase.normalizer.min.region.count";
+  public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3;
+
   protected MasterServices masterServices;
   protected MasterRpcServices masterRpcServices;
+  protected int minRegionCount;
 
   /**
    * Set the master service.
@@ -52,6 +59,9 @@ public abstract class AbstractRegionNormalizer implements RegionNormalizer {
   @Override
   public void setMasterServices(MasterServices masterServices) {
     this.masterServices = masterServices;
+    minRegionCount = masterServices.getConfiguration().getInt(
+        HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY,
+        HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT);
   }
 
   @Override

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java
@@ -23,10 +23,10 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -63,13 +63,13 @@
 public class MergeNormalizer extends AbstractRegionNormalizer {
   private static final Logger LOG = LoggerFactory.getLogger(MergeNormalizer.class);
 
-  private int minRegionCount;
   private int minRegionAge;
   private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
 
-  public MergeNormalizer() {
-    Configuration conf = HBaseConfiguration.create();
-    minRegionCount = conf.getInt("hbase.normalizer.min.region.count", 3);
+  @Override
+  public void setMasterServices(MasterServices masterServices) {
+    super.setMasterServices(masterServices);
+    Configuration conf = masterServices.getConfiguration();
     minRegionAge = conf.getInt("hbase.normalizer.min.region.merge.age", 3);
   }
 

diff --git a/...erver/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/...erver/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -22,7 +22,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import org.apache.hadoop.hbase.HBaseConfiguration;
+
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.RegionInfo;
@@ -51,13 +51,8 @@
 public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
 
   private static final Logger LOG = LoggerFactory.getLogger(SimpleRegionNormalizer.class);
-  private int minRegionCount;
   private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
 
-  public SimpleRegionNormalizer() {
-    minRegionCount = HBaseConfiguration.create().getInt("hbase.normalizer.min.region.count", 3);
-  }
-
   @Override
   public void planSkipped(RegionInfo hri, PlanType type) {
     skippedCount[type.ordinal()]++;
@@ -112,11 +107,7 @@ public List<NormalizationPlan> computePlanForTable(TableName table) throws HBase
     List<RegionInfo> tableRegions =
         masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table);
 
-    if (tableRegions == null || tableRegions.size() < minRegionCount) {
-      int nrRegions = tableRegions == null ? 0 : tableRegions.size();
-      LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is "
-          + "{}, not running normalizer",
-        table, nrRegions, minRegionCount);
+    if (tableRegions == null) {
       return null;
     }
 
@@ -131,9 +122,15 @@ public List<NormalizationPlan> computePlanForTable(TableName table) throws HBase
     }
 
     if (mergeEnabled) {
-      List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
-      if (mergePlans != null) {
-        plans.addAll(mergePlans);
+      if (tableRegions.size() < minRegionCount) {
+        LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run" +
+                " is {}, not running normalizer",
+            table, tableRegions.size(), minRegionCount);
+      } else {
+        List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
+        if (mergePlans != null) {
+          plans.addAll(mergePlans);
+        }
       }
     }
     if (plans.isEmpty()) {

diff --git a/...r/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/...r/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -19,6 +19,7 @@
 
 import static java.lang.String.format;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -30,7 +31,10 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.RegionMetrics;
 import org.apache.hadoop.hbase.ServerName;
@@ -526,6 +530,45 @@ public void testSplitWithTargetRegionSize() throws Exception {
     assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion());
   }
 
+  @Test
+  public void testSplitIfTooFewRegions() throws HBaseIOException {
+    final TableName tableName = TableName.valueOf(name.getMethodName());
+    List<RegionInfo> RegionInfo = new ArrayList<>();
+    Map<byte[], Integer> regionSizes = new HashMap<>();
+
+    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
+        .setStartKey(Bytes.toBytes("aaa"))
+        .setEndKey(Bytes.toBytes("bbb"))
+        .build();
+    RegionInfo.add(hri1);
+    regionSizes.put(hri1.getRegionName(), 1);
+
+    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
+        .setStartKey(Bytes.toBytes("bbb"))
+        .setEndKey(Bytes.toBytes("ccc"))
+        .build();
+    RegionInfo.add(hri2);
+    regionSizes.put(hri2.getRegionName(), 1);
+    // the third region is huge one
+    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
+        .setStartKey(Bytes.toBytes("ccc"))
+        .setEndKey(Bytes.toBytes("ddd"))
+        .build();
+    RegionInfo.add(hri3);
+    regionSizes.put(hri3.getRegionName(), 10);
+
+    setupMocksForNormalizer(regionSizes, RegionInfo);
+
+    Configuration configuration = HBaseConfiguration.create();
+    configuration.setInt(AbstractRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4);
+    when(masterServices.getConfiguration()).thenReturn(configuration);
+
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
+    assertNotNull(plans);
+    NormalizationPlan plan = plans.get(0);
+    assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo());
+  }
+
   @SuppressWarnings("MockitoCast")
   private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
     List<RegionInfo> RegionInfo) {