HBASE-22980 HRegionPartioner getPartition() method incorrectly partit…

…ions the regions of the table. (#590) Signed-off-by: Michael Stack <stack@apache.org> Signed-off-by: Guangxu Cheng <guangxucheng@gmail.com>
apache · Nov 7, 2019 · d1864ae · d1864ae
1 parent 29c27e3
commit d1864ae
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 2 deletions.
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
@@ -82,7 +82,7 @@ public int getPartition(ImmutableBytesWritable key, V2 value, int numPartitions)
     }
     for (int i = 0; i < this.startKeys.length; i++){
       if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){
-        if (i >= numPartitions-1){
+        if (i >= numPartitions){
           // cover if we have less reduces then regions.
           return (Integer.toString(i).hashCode()
               & Integer.MAX_VALUE) % numPartitions;

diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
@@ -89,7 +89,7 @@ public int getPartition(ImmutableBytesWritable key,
     }
     for (int i = 0; i < this.startKeys.length; i++){
       if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){
-        if (i >= numPartitions-1){
+        if (i >= numPartitions){
           // cover if we have less reduces then regions.
           return (Integer.toString(i).hashCode()
               & Integer.MAX_VALUE) % numPartitions;

diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.testclassification.MapReduceTests;
@@ -77,4 +78,27 @@ public void testHRegionPartitioner() throws Exception {
     assertEquals(1, partitioner.getPartition(writable, 10L, 3));
     assertEquals(0, partitioner.getPartition(writable, 10L, 1));
   }
+
+  @Test
+  public void testHRegionPartitionerMoreRegions() throws Exception {
+
+    byte[][] families = { Bytes.toBytes("familyA"), Bytes.toBytes("familyB") };
+
+    TableName tableName = TableName.valueOf(name.getMethodName());
+    UTIL.createTable(tableName, families, 1, Bytes.toBytes("aa"), Bytes.toBytes("cc"), 5);
+
+    Configuration configuration = UTIL.getConfiguration();
+    int numberOfRegions = MetaTableAccessor.getRegionCount(configuration, tableName);
+    assertEquals(5, numberOfRegions);
+
+    HRegionPartitioner<Long, Long> partitioner = new HRegionPartitioner<>();
+    configuration.set(TableOutputFormat.OUTPUT_TABLE, name.getMethodName());
+    partitioner.setConf(configuration);
+
+    // Get some rowKey for the lastRegion
+    ImmutableBytesWritable writable = new ImmutableBytesWritable(Bytes.toBytes("df"));
+
+    // getPartition should return 4 since number of partition = number of reduces.
+    assertEquals(4, partitioner.getPartition(writable, 10L, 5));
+  }
 }