From 69659207ccbe97db674ad59275dab5e6c4b8df62 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Tue, 23 Jul 2019 15:10:32 +0100 Subject: [PATCH 1/8] HBASE-22707 [HBCK2] MasterRpcServices assigns method should try to reload regions from meta if the passed regions isn't found under AssignmentManager RegionsStateStore --- .../hadoop/hbase/MetaTableAccessor.java | 21 ++++ .../hbase/master/MasterRpcServices.java | 4 +- .../master/assignment/AssignmentManager.java | 117 +++++++++++------- .../master/assignment/RegionStateStore.java | 17 +++ 4 files changed, 115 insertions(+), 44 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 3ce67b7ff248..6703de8bd4e0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -59,6 +59,8 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SubstringComparator; import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.master.RegionState; @@ -372,6 +374,25 @@ public static Result getRegionResult(Connection connection, return get(getMetaHTable(connection), get); } + /** + * Scans META table for a row whose key contains the specified regionEncodedName, + * returning a single related Result instance if any row is found, null otherwise. + * + * @param connection the connection to query META table. + * @param regionEncodedName the region encoded name to look for at META. + * @return Result instance with the row related info in META, null otherwise. + * @throws IOException if any errors occur while querying META. + */ + public static Result scanByRegionEncodedName(Connection connection, + String regionEncodedName) throws IOException { + RowFilter rowFilter = new RowFilter(CompareOperator.EQUAL, + new SubstringComparator(regionEncodedName)); + Scan scan = getMetaScan(connection, 1); + scan.setFilter(rowFilter); + ResultScanner resultScanner = getMetaHTable(connection).getScanner(scan); + return resultScanner.next(); + } + /** * Get regions from the merge qualifier of the specified merged region * @return null if it doesn't contain merge qualifier, else two merge regions diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index d3fb1ca8ef6e..2f716fb6b18b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -2408,7 +2408,9 @@ private RegionInfo getRegionInfo(HBaseProtos.RegionSpecifier rs) throws UnknownR String encodedRegionName = Bytes.toString(rs.getValue().toByteArray()); RegionState regionState = this.master.getAssignmentManager().getRegionStates(). getRegionState(encodedRegionName); - ri = regionState == null? null: regionState.getRegion(); + ri = regionState == null ? + this.master.getAssignmentManager().loadRegionFromMeta(encodedRegionName) : + regionState.getRegion(); break; default: break; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index a71e85d4391a..9c4da0f458d9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1383,51 +1383,82 @@ public void processOfflineRegions() { } } - private void loadMeta() throws IOException { - // TODO: use a thread pool - regionStateStore.visitMeta(new RegionStateStore.RegionStateVisitor() { - @Override - public void visitRegionState(Result result, final RegionInfo regionInfo, final State state, - final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) { - if (state == null && regionLocation == null && lastHost == null && - openSeqNum == SequenceId.NO_SEQUENCE_ID) { - // This is a row with nothing in it. - LOG.warn("Skipping empty row={}", result); - return; - } - State localState = state; - if (localState == null) { - // No region state column data in hbase:meta table! Are I doing a rolling upgrade from - // hbase1 to hbase2? Am I restoring a SNAPSHOT or otherwise adding a region to hbase:meta? - // In any of these cases, state is empty. For now, presume OFFLINE but there are probably - // cases where we need to probe more to be sure this correct; TODO informed by experience. - LOG.info(regionInfo.getEncodedName() + " regionState=null; presuming " + State.OFFLINE); - localState = State.OFFLINE; - } - RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); - // Do not need to lock on regionNode, as we can make sure that before we finish loading - // meta, all the related procedures can not be executed. The only exception is for meta - // region related operations, but here we do not load the informations for meta region. - regionNode.setState(localState); - regionNode.setLastHost(lastHost); - regionNode.setRegionLocation(regionLocation); - regionNode.setOpenSeqNum(openSeqNum); - - // Note: keep consistent with other methods, see region(Opening|Opened|Closing) - // RIT/ServerCrash handling should take care of the transiting regions. - if (localState.matches(State.OPEN, State.OPENING, State.CLOSING, State.SPLITTING, - State.MERGING)) { - assert regionLocation != null : "found null region location for " + regionNode; - regionStates.addRegionToServer(regionNode); - } else if (localState == State.OFFLINE || regionInfo.isOffline()) { - regionStates.addToOfflineRegions(regionNode); - } - if (regionNode.getProcedure() != null) { - regionNode.getProcedure().stateLoaded(AssignmentManager.this, regionNode); - } + private class RegionMetaLoadingVisitor implements RegionStateStore.RegionStateVisitor { + + int totalLoadedRegions; + + @Override + public void visitRegionState(Result result, final RegionInfo regionInfo, final State state, + final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) { + if (state == null && regionLocation == null && lastHost == null && + openSeqNum == SequenceId.NO_SEQUENCE_ID) { + // This is a row with nothing in it. + LOG.warn("Skipping empty row={}", result); + return; } - }); + State localState = state; + if (localState == null) { + // No region state column data in hbase:meta table! Are I doing a rolling upgrade from + // hbase1 to hbase2? Am I restoring a SNAPSHOT or otherwise adding a region to hbase:meta? + // In any of these cases, state is empty. For now, presume OFFLINE but there are probably + // cases where we need to probe more to be sure this correct; TODO informed by experience. + LOG.info(regionInfo.getEncodedName() + " regionState=null; presuming " + State.OFFLINE); + localState = State.OFFLINE; + } + RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); + totalLoadedRegions++; + // Do not need to lock on regionNode, as we can make sure that before we finish loading + // meta, all the related procedures can not be executed. The only exception is for meta + // region related operations, but here we do not load the informations for meta region. + regionNode.setState(localState); + regionNode.setLastHost(lastHost); + regionNode.setRegionLocation(regionLocation); + regionNode.setOpenSeqNum(openSeqNum); + + // Note: keep consistent with other methods, see region(Opening|Opened|Closing) + // RIT/ServerCrash handling should take care of the transiting regions. + if (localState.matches(State.OPEN, State.OPENING, State.CLOSING, State.SPLITTING, + State.MERGING)) { + assert regionLocation != null : "found null region location for " + regionNode; + regionStates.addRegionToServer(regionNode); + } else if (localState == State.OFFLINE || regionInfo.isOffline()) { + regionStates.addToOfflineRegions(regionNode); + } + if (regionNode.getProcedure() != null) { + regionNode.getProcedure().stateLoaded(AssignmentManager.this, regionNode); + } + } + }; + /** + * Query META if the given RegionInfo exists, adding to + * AssignmentManager.regionStateStore cache if the region is found in META. + * @param regionEncodedName encoded name for the region to be loaded from META into + * AssignmentManager.regionStateStore cache + * @return RegionInfo instance for the given region if it is present in META + * and got successfully loaded into AssignmentManager.regionStateStore + * cache. + * @throws UnknownRegionException if any errors occur while querying meta or if given + * region couldn't be found in META. + */ + public RegionInfo loadRegionFromMeta(String regionEncodedName) throws UnknownRegionException { + try { + RegionMetaLoadingVisitor visitor = new RegionMetaLoadingVisitor(); + regionStateStore.visitMetaForRegion(regionEncodedName, visitor); + RegionInfo regionInfo = regionStates.getRegionState(regionEncodedName).getRegion(); + if(regionInfo==null){ + throw new UnknownRegionException("Region not found in META."); + } + return regionInfo; + }catch(IOException e){ + LOG.error("Error trying to load region {} from META", regionEncodedName, e); + throw new UnknownRegionException("Error while trying load region from meta"); + } + } + + private void loadMeta() throws IOException { + // TODO: use a thread pool + regionStateStore.visitMeta(new RegionMetaLoadingVisitor()); // every assignment is blocked until meta is loaded. wakeMetaLoadedEvent(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index fdc0a1f0e32b..77b8d9692417 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -96,6 +96,23 @@ public boolean visit(final Result r) throws IOException { }); } + /** + * Queries META table for the passed region encoded name, + * delegating action upon results to the RegionStateVisitor + * passed as second parameter. + * @param regionEncodedName encoded name for the Region we want to query META for. + * @param visitor The RegionStateVisitor instance to react over the query results. + * @throws IOException If some error occurs while querying META or parsing results. + */ + public void visitMetaForRegion(final String regionEncodedName, final RegionStateVisitor visitor) + throws IOException { + Result result = MetaTableAccessor. + scanByRegionEncodedName(master.getConnection(), regionEncodedName); + if(result != null) { + visitMetaEntry(visitor, result); + } + } + private void visitMetaEntry(final RegionStateVisitor visitor, final Result result) throws IOException { final RegionLocations rl = MetaTableAccessor.getRegionLocations(result); From a42cf36fdd7c5049b93f80db61b9ad934a7bda6a Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Tue, 23 Jul 2019 15:15:55 +0100 Subject: [PATCH 2/8] HBASE-22707 [HBCK2] MasterRpcServices assigns method should try to reload regions from meta if the passed regions isn't found under AssignmentManager RegionsStateStore --- .../hadoop/hbase/master/assignment/AssignmentManager.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 9c4da0f458d9..5f36076fa95a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1385,8 +1385,6 @@ public void processOfflineRegions() { private class RegionMetaLoadingVisitor implements RegionStateStore.RegionStateVisitor { - int totalLoadedRegions; - @Override public void visitRegionState(Result result, final RegionInfo regionInfo, final State state, final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) { @@ -1406,7 +1404,6 @@ public void visitRegionState(Result result, final RegionInfo regionInfo, final S localState = State.OFFLINE; } RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); - totalLoadedRegions++; // Do not need to lock on regionNode, as we can make sure that before we finish loading // meta, all the related procedures can not be executed. The only exception is for meta // region related operations, but here we do not load the informations for meta region. From 54e52c35477d7c24fa3a30bae7d57d2fe3c210d3 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 24 Jul 2019 15:36:34 +0100 Subject: [PATCH 3/8] Fixes to address UT failures --- .../hbase/master/assignment/AssignmentManager.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 5f36076fa95a..a0c0c20e028d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1434,18 +1434,15 @@ public void visitRegionState(Result result, final RegionInfo regionInfo, final S * AssignmentManager.regionStateStore cache * @return RegionInfo instance for the given region if it is present in META * and got successfully loaded into AssignmentManager.regionStateStore - * cache. - * @throws UnknownRegionException if any errors occur while querying meta or if given - * region couldn't be found in META. + * cache, null otherwise. + * @throws UnknownRegionException if any errors occur while querying meta. */ public RegionInfo loadRegionFromMeta(String regionEncodedName) throws UnknownRegionException { try { RegionMetaLoadingVisitor visitor = new RegionMetaLoadingVisitor(); regionStateStore.visitMetaForRegion(regionEncodedName, visitor); - RegionInfo regionInfo = regionStates.getRegionState(regionEncodedName).getRegion(); - if(regionInfo==null){ - throw new UnknownRegionException("Region not found in META."); - } + RegionInfo regionInfo = regionStates.getRegionState(regionEncodedName) == null ? null : + regionStates.getRegionState(regionEncodedName).getRegion(); return regionInfo; }catch(IOException e){ LOG.error("Error trying to load region {} from META", regionEncodedName, e); From 5a943435731c0e36fa4afaec9f2f1ba59cc4ae21 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 24 Jul 2019 19:25:27 +0100 Subject: [PATCH 4/8] addressing checkstyle --- .../hadoop/hbase/master/assignment/AssignmentManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index a0c0c20e028d..26736abae26a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1387,7 +1387,7 @@ private class RegionMetaLoadingVisitor implements RegionStateStore.RegionStateVi @Override public void visitRegionState(Result result, final RegionInfo regionInfo, final State state, - final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) { + final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) { if (state == null && regionLocation == null && lastHost == null && openSeqNum == SequenceId.NO_SEQUENCE_ID) { // This is a row with nothing in it. From fb54acc421bb5b5aa3819881326e5aacdc8b02c2 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Sat, 27 Jul 2019 16:10:14 +0100 Subject: [PATCH 5/8] Addressing PR review suggestions --- .../hbase/master/assignment/AssignmentManager.java | 12 +++++++++--- .../hbase/master/assignment/RegionStateStore.java | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 26736abae26a..68a89c949600 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1383,6 +1383,13 @@ public void processOfflineRegions() { } } + /* AM internal RegionStateStore.RegionStateVisitor implementation. To be used when + * scanning META table for region rows, using RegionStateStore utility methods. RegionStateStore + * methods will convert Result into proper RegionInfo instances, but those would still need to be + * added into AssignmentManager.regionStates in-memory cache. + * RegionMetaLoadingVisitor.visitRegionState method provides the logic for adding RegionInfo + * instances as loaded from latest META scan into AssignmentManager.regionStates. + */ private class RegionMetaLoadingVisitor implements RegionStateStore.RegionStateVisitor { @Override @@ -1441,10 +1448,9 @@ public RegionInfo loadRegionFromMeta(String regionEncodedName) throws UnknownReg try { RegionMetaLoadingVisitor visitor = new RegionMetaLoadingVisitor(); regionStateStore.visitMetaForRegion(regionEncodedName, visitor); - RegionInfo regionInfo = regionStates.getRegionState(regionEncodedName) == null ? null : + return regionStates.getRegionState(regionEncodedName) == null ? null : regionStates.getRegionState(regionEncodedName).getRegion(); - return regionInfo; - }catch(IOException e){ + } catch(IOException e) { LOG.error("Error trying to load region {} from META", regionEncodedName, e); throw new UnknownRegionException("Error while trying load region from meta"); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index 77b8d9692417..afeeec584ec9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -108,7 +108,7 @@ public void visitMetaForRegion(final String regionEncodedName, final RegionState throws IOException { Result result = MetaTableAccessor. scanByRegionEncodedName(master.getConnection(), regionEncodedName); - if(result != null) { + if (result != null) { visitMetaEntry(visitor, result); } } From f2e4ecbcdad481ffc3d24720f3b95509fc1f25d7 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Sat, 27 Jul 2019 21:56:10 +0100 Subject: [PATCH 6/8] UTs for each of the new public methods added on this PR. --- .../hadoop/hbase/MetaTableAccessor.java | 2 +- .../master/assignment/RegionStateStore.java | 2 +- .../hadoop/hbase/TestMetaTableAccessor.java | 22 ++++++++++ .../assignment/TestAssignmentManager.java | 41 +++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 6703de8bd4e0..6697830e5e60 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -384,7 +384,7 @@ public static Result getRegionResult(Connection connection, * @throws IOException if any errors occur while querying META. */ public static Result scanByRegionEncodedName(Connection connection, - String regionEncodedName) throws IOException { + String regionEncodedName) throws IOException { RowFilter rowFilter = new RowFilter(CompareOperator.EQUAL, new SubstringComparator(regionEncodedName)); Scan scan = getMetaScan(connection, 1); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index afeeec584ec9..1b927f8589ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -105,7 +105,7 @@ public boolean visit(final Result r) throws IOException { * @throws IOException If some error occurs while querying META or parsing results. */ public void visitMetaForRegion(final String regionEncodedName, final RegionStateVisitor visitor) - throws IOException { + throws IOException { Result result = MetaTableAccessor. scanByRegionEncodedName(master.getConnection(), regionEncodedName); if (result != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java index 1a22cd0e6ad8..62c5ecb23e70 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.ipc.PriorityFunction; import org.apache.hadoop.hbase.ipc.RpcScheduler; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.regionserver.SimpleRpcSchedulerFactory; @@ -890,5 +891,26 @@ public void testEmptyMetaDaughterLocationDuringSplit() throws IOException { } } } + + @Test + public void testScanByRegionEncodedNameExistingRegion() throws Exception { + final TableName tableName = TableName.valueOf("testScanByRegionEncodedNameExistingRegion"); + UTIL.createTable(tableName, "cf"); + final List regions = UTIL.getHBaseCluster().getRegions(tableName); + final String encodedName = regions.get(0).getRegionInfo().getEncodedName(); + final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName); + assertNotNull(result); + assertTrue(result.advance()); + final String resultingRowKey = CellUtil.getCellKeyAsString(result.current()); + assertTrue(resultingRowKey.contains(encodedName)); + UTIL.deleteTable(tableName); + } + + @Test + public void testScanByRegionEncodedNameNonExistingRegion() throws Exception { + final String encodedName = "nonexistingregion"; + final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName); + assertNull(result); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java index 30a57320ae3c..0f4e97fd7532 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java @@ -18,12 +18,14 @@ package org.apache.hadoop.hbase.master.assignment; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; @@ -285,4 +287,43 @@ public void testReopen() throws Exception { assertEquals(reopenFailedCount, reopenProcMetrics.getFailedCounter().getCount()); assertCloseThenOpen(); } + + @Test + public void testLoadRegionFromMetaAfterRegionManuallyAdded() throws Exception { + try { + this.util.startMiniCluster(); + final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager(); + final TableName tableName = TableName. + valueOf("testLoadRegionFromMetaAfterRegionManuallyAdded"); + this.util.createTable(tableName, "f"); + RegionInfo hri = createRegionInfo(tableName, 1); + assertNull("RegionInfo was just instantiated by the test, but " + + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri)); + MetaTableAccessor.addRegionToMeta(this.util.getConnection(), hri); + assertNull("RegionInfo was manually added in META, but " + + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri)); + hri = am.loadRegionFromMeta(hri.getEncodedName()); + assertEquals(hri.getEncodedName(), + am.getRegionStates().getRegionState(hri).getRegion().getEncodedName()); + }finally { + this.util.killMiniHBaseCluster(); + } + } + + @Test + public void testLoadRegionFromMetaRegionNotInMeta() throws Exception { + try { + this.util.startMiniCluster(); + final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager(); + final TableName tableName = TableName.valueOf("testLoadRegionFromMetaRegionNotInMeta"); + this.util.createTable(tableName, "f"); + final RegionInfo hri = createRegionInfo(tableName, 1); + assertNull("RegionInfo was just instantiated by the test, but " + + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri)); + assertNull("RegionInfo was never added in META, should had returned null.", + am.loadRegionFromMeta(hri.getEncodedName())); + }finally { + this.util.killMiniHBaseCluster(); + } + } } From 688cb9eb80cceae5294f09de4dcc3d9320baaca4 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Sat, 27 Jul 2019 21:57:26 +0100 Subject: [PATCH 7/8] missed this new test class on previous commit --- .../assignment/TestRegionStateStore.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java new file mode 100644 index 000000000000..437374976bbb --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.assignment; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestRegionStateStore { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionStateStore.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestRegionStateStore.class); + + protected HBaseTestingUtility util; + + @Before + public void setup() throws Exception { + util = new HBaseTestingUtility(); + util.startMiniCluster(); + } + + @After + public void tearDown() throws Exception { + util.shutdownMiniCluster(); + } + + @Test + public void testVisitMetaForRegionExistingRegion() throws Exception { + final TableName tableName = TableName.valueOf("testVisitMetaForRegion"); + util.createTable(tableName, "cf"); + final List regions = util.getHBaseCluster().getRegions(tableName); + final String encodedName = regions.get(0).getRegionInfo().getEncodedName(); + final RegionStateStore regionStateStore = util.getHBaseCluster().getMaster(). + getAssignmentManager().getRegionStateStore(); + final AtomicBoolean visitorCalled = new AtomicBoolean(false); + regionStateStore.visitMetaForRegion(encodedName, new RegionStateStore.RegionStateVisitor() { + @Override + public void visitRegionState(Result result, RegionInfo regionInfo, RegionState.State state, + ServerName regionLocation, ServerName lastHost, long openSeqNum) { + assertEquals(encodedName, regionInfo.getEncodedName()); + visitorCalled.set(true); + } + }); + assertTrue("Visitor has not been called.", visitorCalled.get()); + } + + @Test + public void testVisitMetaForRegionNonExistingRegion() throws Exception { + final String encodedName = "fakeencodedregionname"; + final RegionStateStore regionStateStore = util.getHBaseCluster().getMaster(). + getAssignmentManager().getRegionStateStore(); + final AtomicBoolean visitorCalled = new AtomicBoolean(false); + regionStateStore.visitMetaForRegion(encodedName, new RegionStateStore.RegionStateVisitor() { + @Override + public void visitRegionState(Result result, RegionInfo regionInfo, RegionState.State state, + ServerName regionLocation, ServerName lastHost, long openSeqNum) { + visitorCalled.set(true); + } + }); + assertFalse("Visitor has been called, but it shouldn't.", visitorCalled.get()); + } + +} From 57a8371b01a47166980874d33512d0b94f6544f9 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 31 Jul 2019 17:42:59 +0100 Subject: [PATCH 8/8] addressing checkstyles --- .../org/apache/hadoop/hbase/TestMetaTableAccessor.java | 9 ++++++--- .../hbase/master/assignment/TestRegionStateStore.java | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java index 62c5ecb23e70..15e276d6d117 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java @@ -867,7 +867,8 @@ public void testEmptyMetaDaughterLocationDuringSplit() throws IOException { List regionInfos = Lists.newArrayList(parent); MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); - MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, serverName0, 3); + MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, + serverName0, 3); Get get1 = new Get(splitA.getRegionName()); Result resultA = meta.get(get1); Cell serverCellA = resultA.getColumnLatestCell(HConstants.CATALOG_FAMILY, @@ -898,7 +899,8 @@ public void testScanByRegionEncodedNameExistingRegion() throws Exception { UTIL.createTable(tableName, "cf"); final List regions = UTIL.getHBaseCluster().getRegions(tableName); final String encodedName = regions.get(0).getRegionInfo().getEncodedName(); - final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName); + final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), + encodedName); assertNotNull(result); assertTrue(result.advance()); final String resultingRowKey = CellUtil.getCellKeyAsString(result.current()); @@ -909,7 +911,8 @@ public void testScanByRegionEncodedNameExistingRegion() throws Exception { @Test public void testScanByRegionEncodedNameNonExistingRegion() throws Exception { final String encodedName = "nonexistingregion"; - final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName); + final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), + encodedName); assertNull(result); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java index 437374976bbb..b4af3708f7c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java @@ -20,6 +20,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.ServerName; @@ -38,8 +41,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; @Category({ MasterTests.class, MediumTests.class }) public class TestRegionStateStore {