From 636fa2c6b02d67427c2c33f71b8e9977e064debb Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 27 Nov 2019 08:41:23 +0000 Subject: [PATCH] =?UTF-8?q?HBASE-23313=20[hbck2]=20setRegionState=20should?= =?UTF-8?q?=20update=20Master=20in-memory=20sta=E2=80=A6=20(#864)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mingliang Liu Signed-off-by: stack --- .../apache/hadoop/hbase/client/HBaseHbck.java | 45 ++++++++++++++----- .../org/apache/hadoop/hbase/client/Hbck.java | 9 ++++ .../hbase/client/RegionInfoBuilder.java | 5 +++ .../hbase/shaded/protobuf/ProtobufUtil.java | 12 +++++ .../shaded/protobuf/RequestConverter.java | 14 ++++++ .../src/main/protobuf/HBase.proto | 1 + .../src/main/protobuf/Master.proto | 12 +++++ .../hbase/master/MasterRpcServices.java | 40 +++++++++++++++++ .../apache/hadoop/hbase/client/TestHbck.java | 35 +++++++++++++++ 9 files changed, 162 insertions(+), 11 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java index f2bfac02b4c1..651922ea7c96 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java @@ -18,22 +18,19 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.concurrent.Callable; import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.ipc.RpcControllerFactory; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ipc.RpcControllerFactory; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BypassProcedureRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BypassProcedureResponse; @@ -45,6 +42,13 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ScheduleServerCrashProcedureResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignsResponse; +import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; + +import org.apache.yetus.audience.InterfaceAudience; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Use {@link Connection#getHbck()} to obtain an instance of {@link Hbck} instead of * constructing an HBaseHbck directly. @@ -98,8 +102,8 @@ public boolean isAborted() { public TableState setTableStateInMeta(TableState state) throws IOException { try { GetTableStateResponse response = hbck.setTableStateInMeta( - rpcControllerFactory.newController(), - RequestConverter.buildSetTableStateInMetaRequest(state)); + rpcControllerFactory.newController(), + RequestConverter.buildSetTableStateInMetaRequest(state)); return TableState.convert(state.getTableName(), response.getTableState()); } catch (ServiceException se) { LOG.debug("table={}, state={}", state.getTableName(), state.getState(), se); @@ -107,6 +111,25 @@ public TableState setTableStateInMeta(TableState state) throws IOException { } } + @Override + public List setRegionStateInMeta(List states) throws IOException { + try { + if(LOG.isDebugEnabled()) { + states.forEach(s -> + LOG.debug("region={}, state={}", s.getRegion().getRegionName(), s.getState()) + ); + } + MasterProtos.GetRegionStateInMetaResponse response = hbck.setRegionStateInMeta( + rpcControllerFactory.newController(), + RequestConverter.buildSetRegionStateInMetaRequest(states)); + final List result = new ArrayList<>(); + response.getStatesList().forEach(s -> result.add(RegionState.convert(s))); + return result; + } catch (ServiceException se) { + throw new IOException(se); + } + } + @Override public List assigns(List encodedRegionNames, boolean override) throws IOException { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java index 4f37f130593b..ff982b2ef978 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -54,6 +55,14 @@ public interface Hbck extends Abortable, Closeable { */ TableState setTableStateInMeta(TableState state) throws IOException; + /** + * Update region state in Meta only. No procedures are submitted to manipulate the given region + * or any other region from same table. + * @param states list of all region states to be updated in meta + * @return previous state of the region in Meta + */ + List setRegionStateInMeta(List states) throws IOException; + /** * Like {@link Admin#assign(byte[])} but 'raw' in that it can do more than one Region at a time * -- good if many Regions to online -- and it will schedule the assigns even in the case where diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfoBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfoBuilder.java index cd9e40b95243..e79a7b76cdbf 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfoBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfoBuilder.java @@ -116,6 +116,11 @@ public RegionInfoBuilder setOffline(boolean offLine) { return this; } + public RegionInfoBuilder setEncodedName(String encodedName) { + this.encodedName = encodedName; + return this; + } + public RegionInfo build() { return new MutableRegionInfo(tableName, startKey, endKey, split, regionId, replicaId, offLine, regionName, encodedName); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 94a2805b61cc..c6e48d420148 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -2226,6 +2226,14 @@ public static HBaseProtos.TableName toProtoTableName(TableName tableName) { .setQualifier(UnsafeByteOperations.unsafeWrap(tableName.getQualifier())).build(); } + public static HBaseProtos.RegionInfo toProtoRegionInfo( + org.apache.hadoop.hbase.client.RegionInfo regionInfo) { + return HBaseProtos.RegionInfo.newBuilder() + .setRegionId(regionInfo.getRegionId()) + .setRegionEncodedName(regionInfo.getEncodedName()) + .setTableName(toProtoTableName(regionInfo.getTable())).build(); + } + public static List toTableNameList(List tableNamesList) { if (tableNamesList == null) { return new ArrayList<>(); @@ -3145,6 +3153,7 @@ public static HBaseProtos.RegionInfo toRegionInfo(final org.apache.hadoop.hbase. builder.setOffline(info.isOffline()); builder.setSplit(info.isSplit()); builder.setReplicaId(info.getReplicaId()); + builder.setRegionEncodedName(info.getEncodedName()); return builder.build(); } @@ -3184,6 +3193,9 @@ public static org.apache.hadoop.hbase.client.RegionInfo toRegionInfo(final HBase if (proto.hasOffline()) { rib.setOffline(proto.getOffline()); } + if (proto.hasRegionEncodedName()) { + rib.setEncodedName(proto.getRegionEncodedName()); + } return rib.build(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 972f9811c19f..075829c2e391 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.SyncReplicationState; import org.apache.hadoop.hbase.util.Bytes; @@ -132,6 +133,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetBalancerRunningRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetCleanerChoreRunningRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetRegionStateInMetaRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos .SetSnapshotCleanupRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetSplitOrMergeEnabledRequest; @@ -1258,6 +1260,18 @@ public static SetTableStateInMetaRequest buildSetTableStateInMetaRequest(final T .setTableName(ProtobufUtil.toProtoTableName(state.getTableName())).build(); } + /** + * Creates a protocol buffer SetRegionStateInMetaRequest + * @param states list of regions states to update in Meta + * @return a SetRegionStateInMetaRequest + */ + public static SetRegionStateInMetaRequest buildSetRegionStateInMetaRequest( + final List states) { + final SetRegionStateInMetaRequest.Builder builder = SetRegionStateInMetaRequest.newBuilder(); + states.forEach(s -> builder.addStates(s.convert())); + return builder.build(); + } + /** * Creates a protocol buffer GetTableDescriptorsRequest for a single table * diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto b/hbase-protocol-shaded/src/main/protobuf/HBase.proto index d06bc8b42d3e..cf577e93ef76 100644 --- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto +++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto @@ -79,6 +79,7 @@ message RegionInfo { optional bool offline = 5; optional bool split = 6; optional int32 replica_id = 7 [default = 0]; + optional string region_encoded_name = 8; } /** diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index fee9ab8419d1..69377a62b11b 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -517,6 +517,10 @@ message GetTableStateResponse { required TableState table_state = 1; } +message GetRegionStateInMetaResponse { + repeated RegionState states = 1; +} + message GetClusterStatusRequest { repeated Option options = 1; @@ -1090,6 +1094,10 @@ message SetTableStateInMetaRequest { required TableState table_state = 2; } +message SetRegionStateInMetaRequest { + repeated RegionState states = 2; +} + /** Like Admin's AssignRegionRequest except it can * take one or more Regions at a time. */ @@ -1152,6 +1160,10 @@ service HbckService { rpc SetTableStateInMeta(SetTableStateInMetaRequest) returns(GetTableStateResponse); + /** Update state of the region in meta only*/ + rpc SetRegionStateInMeta(SetRegionStateInMetaRequest) + returns(GetRegionStateInMetaResponse); + /** * Assign regions. * Like Admin's assign but works even if the diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 92f8178a197e..5ef32dae0108 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.MasterSwitchType; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Table; @@ -197,6 +198,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetProcedureResultResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetProceduresRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetProceduresResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetRegionStateInMetaResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableDescriptorsRequest; @@ -276,6 +278,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetRegionStateInMetaRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos .SetSnapshotCleanupRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos @@ -2465,6 +2468,42 @@ public GetTableStateResponse setTableStateInMeta(RpcController controller, } } + /** + * Update state of the region in meta only. This is required by hbck in some situations to cleanup + * stuck assign/ unassign regions procedures for the table. + * + * @return previous states of the regions + */ + @Override + public GetRegionStateInMetaResponse setRegionStateInMeta(RpcController controller, + SetRegionStateInMetaRequest request) throws ServiceException { + final GetRegionStateInMetaResponse.Builder builder = GetRegionStateInMetaResponse.newBuilder(); + for(ClusterStatusProtos.RegionState s : request.getStatesList()) { + try { + RegionInfo info = this.master.getAssignmentManager(). + loadRegionFromMeta(s.getRegionInfo().getRegionEncodedName()); + LOG.trace("region info loaded from meta table: {}", info); + RegionState prevState = this.master.getAssignmentManager().getRegionStates(). + getRegionState(info); + RegionState newState = RegionState.convert(s); + LOG.info("{} set region={} state from {} to {}", master.getClientIdAuditPrefix(), info, + prevState.getState(), newState.getState()); + Put metaPut = MetaTableAccessor.makePutFromRegionInfo(info, System.currentTimeMillis()); + metaPut.addColumn(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER, + Bytes.toBytes(newState.getState().name())); + List putList = new ArrayList<>(); + putList.add(metaPut); + MetaTableAccessor.putsToMetaTable(this.master.getConnection(), putList); + //Loads from meta again to refresh AM cache with the new region state + this.master.getAssignmentManager().loadRegionFromMeta(info.getEncodedName()); + builder.addStates(prevState.convert()); + } catch (Exception e) { + throw new ServiceException(e); + } + } + return builder.build(); + } + /** * Get RegionInfo from Master using content of RegionSpecifier as key. * @return RegionInfo found by decoding rs or null if none found @@ -2834,4 +2873,5 @@ private boolean shouldSubmitSCP(ServerName serverName) { } return true; } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java index 070cfe0c38b3..a77c687b1377 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java @@ -21,8 +21,11 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; @@ -38,6 +41,7 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface; import org.apache.hadoop.hbase.procedure2.Procedure; @@ -48,6 +52,7 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -182,6 +187,36 @@ public void testSetTableStateInMeta() throws Exception { prevState.isDisabled()); } + @Test + public void testSetRegionStateInMeta() throws Exception { + Hbck hbck = getHbck(); + try(Admin admin = TEST_UTIL.getAdmin()){ + final List regions = admin.getRegions(TABLE_NAME); + final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); + final List prevStates = new ArrayList<>(); + final List newStates = new ArrayList<>(); + final Map> regionsMap = new HashMap<>(); + regions.forEach(r -> { + RegionState prevState = am.getRegionStates().getRegionState(r); + prevStates.add(prevState); + RegionState newState = RegionState.createForTesting(r, RegionState.State.CLOSED); + newStates.add(newState); + regionsMap.put(r.getEncodedName(), new Pair<>(prevState, newState)); + }); + final List result = hbck.setRegionStateInMeta(newStates); + result.forEach(r -> { + RegionState prevState = regionsMap.get(r.getRegion().getEncodedName()).getFirst(); + assertEquals(prevState.getState(), r.getState()); + }); + regions.forEach(r -> { + RegionState cachedState = am.getRegionStates().getRegionState(r.getEncodedName()); + RegionState newState = regionsMap.get(r.getEncodedName()).getSecond(); + assertEquals(newState.getState(), cachedState.getState()); + }); + hbck.setRegionStateInMeta(prevStates); + } + } + @Test public void testAssigns() throws Exception { Hbck hbck = getHbck();