From 4c83067e1e920154648b31d48e81db76127c7da7 Mon Sep 17 00:00:00 2001 From: Reid Chan Date: Wed, 15 Apr 2020 19:04:40 +0800 Subject: [PATCH] HBASE-24112 [RSGroup] Support renaming rsgroup (#1520) Signed-off-by: Duo Zhang Signed-off-by: stack --- .../hadoop/hbase/rsgroup/RSGroupAdmin.java | 7 +++ .../hbase/rsgroup/RSGroupAdminClient.java | 13 ++++++ .../hbase/rsgroup/RSGroupAdminEndpoint.java | 26 +++++++++++ .../hbase/rsgroup/RSGroupAdminServer.java | 7 +++ .../hbase/rsgroup/RSGroupInfoManager.java | 7 +++ .../hbase/rsgroup/RSGroupInfoManagerImpl.java | 17 ++++++++ .../src/main/protobuf/RSGroupAdmin.proto | 11 +++++ .../hbase/rsgroup/TestRSGroupsAdmin1.java | 43 +++++++++++++++++++ .../hbase/rsgroup/TestRSGroupsBase.java | 16 +++++++ .../rsgroup/VerifyingRSGroupAdminClient.java | 6 +++ .../hbase/coprocessor/MasterObserver.java | 22 ++++++++++ .../hbase/master/MasterCoprocessorHost.java | 20 +++++++++ 12 files changed, 195 insertions(+) diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java index 9ea996be1cc3..c389dc5d6cf0 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java @@ -98,4 +98,11 @@ void moveServersAndTables(Set
servers, Set tables, * @param servers set of servers to remove */ void removeServers(Set
servers) throws IOException; + + /** + * Rename rsgroup. + * @param oldName old rsgroup name + * @param newName new rsgroup name + */ + void renameRSGroup(String oldName, String newName) throws IOException; } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java index e8a141064381..bcfe3a4865b6 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RSGroupAdminService; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RenameRSGroupRequest; import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; import org.apache.yetus.audience.InterfaceAudience; @@ -236,4 +237,16 @@ public void removeServers(Set
servers) throws IOException { throw ProtobufUtil.handleRemoteException(e); } } + + @Override + public void renameRSGroup(String oldName, String newName) throws IOException { + RenameRSGroupRequest request = RenameRSGroupRequest.newBuilder() + .setOldRsgroupName(oldName) + .setNewRsgroupName(newName).build(); + try { + stub.renameRSGroup(null, request); + } catch (ServiceException e) { + throw ProtobufUtil.handleRemoteException(e); + } + } } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java index b869e81fc648..fbbd671b4539 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java @@ -79,6 +79,8 @@ import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RenameRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RenameRSGroupResponse; import org.apache.hadoop.hbase.protobuf.generated.TableProtos; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; @@ -441,6 +443,30 @@ public void removeServers(RpcController controller, } done.run(builder.build()); } + + @Override + public void renameRSGroup(RpcController controller, + RenameRSGroupRequest request, + RpcCallback done) { + String oldRSGroup = request.getOldRsgroupName(); + String newRSGroup = request.getNewRsgroupName(); + LOG.info("{} rename rsgroup from {} to {}", + master.getClientIdAuditPrefix(), oldRSGroup, newRSGroup); + + RenameRSGroupResponse.Builder builder = RenameRSGroupResponse.newBuilder(); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preRenameRSGroup(oldRSGroup, newRSGroup); + } + groupAdminServer.renameRSGroup(oldRSGroup, newRSGroup); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postRenameRSGroup(oldRSGroup, newRSGroup); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } } boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java index 9f038ffea934..bf9f8488c82f 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java @@ -525,6 +525,13 @@ public void removeServers(Set
servers) throws IOException { } } + @Override + public void renameRSGroup(String oldName, String newName) throws IOException { + synchronized (rsGroupInfoManager) { + rsGroupInfoManager.renameRSGroup(oldName, newName); + } + } + private Map rsGroupGetRegionsInTransition(String groupName) throws IOException { Map rit = Maps.newTreeMap(); diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java index b0cd5ff57f2b..d2bf5390bfe3 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java @@ -125,4 +125,11 @@ void moveServersAndTables(Set
servers, Set tables, * @param servers set of servers to remove */ void removeServers(Set
servers) throws IOException; + + /** + * Rename RSGroup + * @param oldName old rsgroup name + * @param newName new rsgroup name + */ + void renameRSGroup(String oldName, String newName) throws IOException; } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index c2c463b497da..2f452b3f41a3 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -349,6 +349,23 @@ public synchronized void removeServers(Set
servers) throws IOException } } + @Override + public void renameRSGroup(String oldName, String newName) throws IOException { + checkGroupName(oldName); + checkGroupName(newName); + if (oldName.equals(RSGroupInfo.DEFAULT_GROUP)) { + throw new ConstraintException("Can't rename default rsgroup"); + } + + RSGroupInfo oldGroup = getRSGroup(oldName); + Map newGroupMap = Maps.newHashMap(rsGroupMap); + newGroupMap.remove(oldName); + RSGroupInfo newGroup = new RSGroupInfo(newName, + (SortedSet
) oldGroup.getServers(), oldGroup.getTables()); + newGroupMap.put(newName, newGroup); + flushConfig(newGroupMap); + } + List retrieveGroupListFromGroupTable() throws IOException { List rsGroupInfoList = Lists.newArrayList(); try (Table table = conn.getTable(RSGROUP_TABLE_NAME); diff --git a/hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto b/hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto index 316b06f55c92..10f6570a2f8c 100644 --- a/hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto +++ b/hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto @@ -123,6 +123,14 @@ message RemoveServersRequest { message RemoveServersResponse { } +message RenameRSGroupRequest { + required string old_rsgroup_name = 1; + required string new_rsgroup_name = 2; +} + +message RenameRSGroupResponse { +} + service RSGroupAdminService { rpc GetRSGroupInfo(GetRSGroupInfoRequest) returns (GetRSGroupInfoResponse); @@ -156,4 +164,7 @@ service RSGroupAdminService { rpc RemoveServers(RemoveServersRequest) returns (RemoveServersResponse); + + rpc RenameRSGroup(RenameRSGroupRequest) + returns (RenameRSGroupResponse); } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java index ca53985dc812..60e66bb2da4b 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java @@ -497,4 +497,47 @@ private void toggleQuotaCheckAndRestartMiniCluster(boolean enable) throws Except TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); initialize(); } + + @Test + public void testRenameRSGroup() throws Exception { + // Add rsgroup, and assign 2 servers and a table to it. + RSGroupInfo oldgroup = addGroup("oldgroup", 2); + final TableName tb1 = TableName.valueOf("testRename"); + TEST_UTIL.createTable(tb1, "tr"); + rsGroupAdmin.moveTables(Sets.newHashSet(tb1), oldgroup.getName()); + TEST_UTIL.waitFor(1000, + (Waiter.Predicate) () -> + rsGroupAdmin.getRSGroupInfoOfTable(tb1).getServers().size() == 2); + oldgroup = rsGroupAdmin.getRSGroupInfo(oldgroup.getName()); + assertEquals(2, oldgroup.getServers().size()); + assertEquals(oldgroup.getName(), rsGroupAdmin.getRSGroupInfoOfTable(tb1).getName()); + assertTrue(oldgroup.getTables().contains(tb1)); + + // Another rsgroup and table for verification + // that they are unchanged during we're renaming oldgroup. + RSGroupInfo normal = addGroup("normal", 1); + final TableName tb2 = TableName.valueOf("unmovedTable"); + TEST_UTIL.createTable(tb2, "ut"); + rsGroupAdmin.moveTables(Sets.newHashSet(tb2), normal.getName()); + TEST_UTIL.waitFor(1000, + (Waiter.Predicate) () -> + rsGroupAdmin.getRSGroupInfoOfTable(tb2).getServers().size() == 1); + normal = rsGroupAdmin.getRSGroupInfo(normal.getName()); + assertEquals(1, normal.getServers().size()); + assertEquals(normal.getName(), rsGroupAdmin.getRSGroupInfoOfTable(tb2).getName()); + assertTrue(normal.containsTable(tb2)); + + + // Rename rsgroup + rsGroupAdmin.renameRSGroup(oldgroup.getName(), "newgroup"); + Set
servers = oldgroup.getServers(); + RSGroupInfo newgroup = rsGroupAdmin.getRSGroupInfo("newgroup"); + assertEquals(servers.size(), newgroup.getServers().size()); + for (Address server : servers) { + assertTrue(newgroup.containsServer(server)); + } + assertEquals(newgroup.getName(), rsGroupAdmin.getRSGroupInfoOfTable(tb1).getName()); + assertTrue(newgroup.containsTable(tb1)); + assertEquals(normal.getName(), rsGroupAdmin.getRSGroupInfoOfTable(tb2).getName()); + } } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java index 8d7a67fe106b..6cb738d61367 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java @@ -304,6 +304,8 @@ public static class CPMasterObserver implements MasterCoprocessor, MasterObserve boolean postRemoveServersCalled = false; boolean preMoveServersAndTables = false; boolean postMoveServersAndTables = false; + boolean preReNameRSGroupCalled = false; + boolean postReNameRSGroupCalled = false; public void resetFlags() { preBalanceRSGroupCalled = false; @@ -320,6 +322,8 @@ public void resetFlags() { postRemoveServersCalled = false; preMoveServersAndTables = false; postMoveServersAndTables = false; + preReNameRSGroupCalled = false; + postReNameRSGroupCalled = false; } @Override @@ -412,6 +416,18 @@ public void postBalanceRSGroup(final ObserverContext ctx, + String oldName, String newName) throws IOException { + preReNameRSGroupCalled = true; + } + + @Override + public void postRenameRSGroup(ObserverContext ctx, + String oldName, String newName) throws IOException { + postReNameRSGroupCalled = true; + } } } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java index 2af2721c2e1d..923a77d44f94 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java @@ -119,6 +119,12 @@ public void removeServers(Set
servers) throws IOException { verify(); } + @Override + public void renameRSGroup(String oldName, String newName) throws IOException { + wrapped.renameRSGroup(oldName, newName); + verify(); + } + public void verify() throws IOException { Map groupMap = Maps.newHashMap(); Set zList = Sets.newHashSet(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 69abfc2640e8..48c259bac4e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -1624,4 +1624,26 @@ default void preHasUserPermissions(ObserverContext default void postHasUserPermissions(ObserverContext ctx, String userName, List permissions) throws IOException { } + + /** + * Called before rename rsgroup. + * @param ctx the environment to interact with the framework and master + * @param oldName old rsgroup name + * @param newName new rsgroup name + * @throws IOException on failure + */ + default void preRenameRSGroup(final ObserverContext ctx, + final String oldName, final String newName) throws IOException { + } + + /** + * Called after rename rsgroup. + * @param ctx the environment to interact with the framework and master + * @param oldName old rsgroup name + * @param newName new rsgroup name + * @throws IOException on failure + */ + default void postRenameRSGroup(final ObserverContext ctx, + final String oldName, final String newName) throws IOException { + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 13c67644ea98..236eda0581f9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -1453,6 +1453,26 @@ public void call(MasterObserver observer) throws IOException { }); } + public void preRenameRSGroup(final String oldName, final String newName) + throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null: new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.preRenameRSGroup(this, oldName, newName); + } + }); + } + + public void postRenameRSGroup(final String oldName, final String newName) + throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null: new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.postRenameRSGroup(this, oldName, newName); + } + }); + } + public void preRemoveServers(final Set
servers) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {