From 9b828154f99c2eff8833713fbecb3de14235c00b Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Thu, 5 Jun 2025 17:56:44 +0900 Subject: [PATCH 1/3] HBASE-29378 Make TestingHBaseCluster expose address/port information for end users --- .../hbase/testing/TestingHBaseCluster.java | 20 +++++++++++++ .../testing/TestingHBaseClusterImpl.java | 30 +++++++++++++++++++ .../testing/TestTestingHBaseCluster.java | 20 ++++++++++++- 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java index 1136e660453b..ef38180bc4f4 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java @@ -135,6 +135,26 @@ public interface TestingHBaseCluster { */ List getRegionServerAddresses(); + /** + * Get the info port of the active master if there is one. + */ + Optional getActiveMasterInfoPort(); + + /** + * Get the info port of the active NameNode if there is one. + */ + Optional getActiveNameNodeInfoPort(); + + /** + * Get the client port of the ZooKeeper if there is one. + */ + Optional getActiveZooKeeperClientPort(); + + /** + * Get the list of master addresses. + */ + List getMasterAddresses(); + /** * Get the server side {@link Region} interface for the specific region. *

diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java index 35198e770db0..3dbe78f5ed21 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java @@ -24,6 +24,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hbase.HBaseTestingUtil; @@ -37,6 +38,8 @@ import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; @@ -219,6 +222,33 @@ public List getRegionServerAddresses() { .map(t -> t.getRegionServer().getServerName()).collect(Collectors.toList()); } + @Override + public Optional getActiveMasterInfoPort() { + return Optional.ofNullable(util.getMiniHBaseCluster().getMaster()) + .map(m -> m.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0)) + .filter(port -> port > 0); + } + + @Override + public Optional getActiveNameNodeInfoPort() { + return Stream.of(util.getDFSCluster().getNameNodeInfos()).map(i -> i.nameNode) + .filter(NameNode::isActiveState).map(nn -> nn.getHttpAddress().getPort()) + .filter(port -> port > 0).findFirst(); + } + + @Override + public Optional getActiveZooKeeperClientPort() { + return Optional.ofNullable(util.getZkCluster()).map(MiniZooKeeperCluster::getClientPort) + .filter(port -> port > 0); + } + + @Override + public List getMasterAddresses() { + return util.getMiniHBaseCluster().getMasterThreads().stream() + .map(mt -> mt.getMaster().getServerName()).map(sn -> sn.getHostname() + ':' + sn.getPort()) + .collect(Collectors.toList()); + } + @Override public Optional getRegion(RegionInfo regionInfo) { for (RegionServerThread t : util.getMiniHBaseCluster().getRegionServerThreads()) { diff --git a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java index 874e7562a23f..f4d52312bd1b 100644 --- a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java +++ b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java @@ -23,7 +23,11 @@ import static org.junit.Assert.assertTrue; import java.util.Collection; +import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; @@ -59,8 +63,10 @@ public class TestTestingHBaseCluster { @BeforeClass public static void setUpBeforeClass() throws Exception { + final Configuration conf = HBaseConfiguration.create(); + conf.setInt(HConstants.MASTER_INFO_PORT, 0); CLUSTER = TestingHBaseCluster.create(TestingHBaseClusterOption.builder().numMasters(2) - .numRegionServers(3).numDataNodes(3).build()); + .numRegionServers(3).numDataNodes(3).conf(conf).build()); } @AfterClass @@ -150,5 +156,17 @@ public void testGetAddresses() throws Exception { assertTrue(CLUSTER.getActiveMasterAddress().isPresent()); assertEquals(1, CLUSTER.getBackupMasterAddresses().size()); assertEquals(3, CLUSTER.getRegionServerAddresses().size()); + + // [HOSTNAME1:PORT1, HOSTNAME2:PORT2] + final List masterAddrs = CLUSTER.getMasterAddresses(); + assertEquals(2, masterAddrs.size()); + assertTrue(masterAddrs.stream().allMatch(addr -> addr.matches(".*:[0-9]+$"))); + } + + @Test + public void testGetPorts() throws Exception { + assertTrue(CLUSTER.getActiveMasterInfoPort().isPresent()); + assertTrue(CLUSTER.getActiveNameNodeInfoPort().isPresent()); + assertTrue(CLUSTER.getActiveZooKeeperClientPort().isPresent()); } } From 86ca728b3baef1263a12896bfa51dfec86c02ffb Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Fri, 6 Jun 2025 23:12:42 +0900 Subject: [PATCH 2/3] Change method signatures --- .../hbase/testing/TestingHBaseCluster.java | 12 ++-- .../testing/TestingHBaseClusterImpl.java | 69 ++++++++++++++++--- .../testing/TestTestingHBaseCluster.java | 10 ++- 3 files changed, 71 insertions(+), 20 deletions(-) diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java index ef38180bc4f4..c6b5952585e0 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseCluster.java @@ -136,19 +136,19 @@ public interface TestingHBaseCluster { List getRegionServerAddresses(); /** - * Get the info port of the active master if there is one. + * Get the web UI address of the active master if there is one. */ - Optional getActiveMasterInfoPort(); + Optional getActiveMasterInfoAddress(); /** - * Get the info port of the active NameNode if there is one. + * Get the web UI address of the active NameNode if there is one. */ - Optional getActiveNameNodeInfoPort(); + Optional getActiveNameNodeInfoAddress(); /** - * Get the client port of the ZooKeeper if there is one. + * Get the 'hbase.zookeeper.quorum' configuration value for the ZooKeeper cluster. */ - Optional getActiveZooKeeperClientPort(); + Optional getZooKeeperQuorum(); /** * Get the list of master addresses. diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java index 3dbe78f5ed21..32215773f0c4 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.testing; +import static org.apache.hadoop.hbase.http.ServerConfigurationKeys.HBASE_SSL_ENABLED_KEY; + +import java.net.InetSocketAddress; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -39,6 +42,7 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.yetus.audience.InterfaceAudience; @@ -223,23 +227,66 @@ public List getRegionServerAddresses() { } @Override - public Optional getActiveMasterInfoPort() { - return Optional.ofNullable(util.getMiniHBaseCluster().getMaster()) - .map(m -> m.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0)) - .filter(port -> port > 0); + public Optional getActiveMasterInfoAddress() { + final HMaster master = util.getMiniHBaseCluster().getMaster(); + + // No active master + if (master == null) { + return Optional.empty(); + } + + final Configuration conf = master.getConfiguration(); + final int port = conf.getInt(HConstants.MASTER_INFO_PORT, 0); + + // Web UI disabled + if (port <= 0) { + return Optional.empty(); + } + + final String protocol = conf.getBoolean(HBASE_SSL_ENABLED_KEY, false) ? "https" : "http"; + final String hostname = master.getServerName().getHostname(); + return Optional.of(String.format("%s://%s:%d", protocol, hostname, port)); } @Override - public Optional getActiveNameNodeInfoPort() { - return Stream.of(util.getDFSCluster().getNameNodeInfos()).map(i -> i.nameNode) - .filter(NameNode::isActiveState).map(nn -> nn.getHttpAddress().getPort()) - .filter(port -> port > 0).findFirst(); + public Optional getActiveNameNodeInfoAddress() { + final MiniDFSCluster cluster = util.getDFSCluster(); + + // External DFS cluster + if (cluster == null) { + return Optional.empty(); + } + + final NameNode ann = Stream.of(cluster.getNameNodeInfos()).map(i -> i.nameNode) + .filter(NameNode::isActiveState).findFirst().orElse(null); + + // No active NameNode + if (ann == null) { + return Optional.empty(); + } + + String protocol = "http"; + InetSocketAddress address = ann.getHttpAddress(); + if (address == null) { + protocol = "https"; + address = ann.getHttpsAddress(); + } + + // Neither HTTP nor HTTPS address is available + if (address == null) { + return Optional.empty(); + } + + return Optional + .of(String.format("%s://%s:%d", protocol, address.getHostName(), address.getPort())); } @Override - public Optional getActiveZooKeeperClientPort() { - return Optional.ofNullable(util.getZkCluster()).map(MiniZooKeeperCluster::getClientPort) - .filter(port -> port > 0); + public Optional getZooKeeperQuorum() { + return Optional.ofNullable(util.getZkCluster()) + .map(zk -> zk.getClientPortList().stream() + .map(port -> String.format("%s:%d", MiniZooKeeperCluster.HOST, port)) + .collect(Collectors.joining(","))); } @Override diff --git a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java index f4d52312bd1b..81f9d279a226 100644 --- a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java +++ b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.List; +import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -165,8 +166,11 @@ public void testGetAddresses() throws Exception { @Test public void testGetPorts() throws Exception { - assertTrue(CLUSTER.getActiveMasterInfoPort().isPresent()); - assertTrue(CLUSTER.getActiveNameNodeInfoPort().isPresent()); - assertTrue(CLUSTER.getActiveZooKeeperClientPort().isPresent()); + final String addressPattern = "^https?://.*:[0-9]+$"; + assertTrue(CLUSTER.getActiveMasterInfoAddress().map(a -> Pattern.matches(addressPattern, a)) + .orElse(false)); + assertTrue(CLUSTER.getActiveNameNodeInfoAddress().map(a -> Pattern.matches(addressPattern, a)) + .orElse(false)); + assertTrue(CLUSTER.getZooKeeperQuorum().isPresent()); } } From 5bc0f4c3e5aae08045a11fef12c6a9f50aa0ee85 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Mon, 9 Jun 2025 09:51:10 +0900 Subject: [PATCH 3/3] Fix numZkServers ignored --- .../hbase/testing/TestingHBaseClusterImpl.java | 2 +- .../hbase/testing/TestTestingHBaseCluster.java | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java index 32215773f0c4..95f4ffff3ac8 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java @@ -154,7 +154,7 @@ public void startHBaseCluster() throws Exception { public void start() throws Exception { Preconditions.checkState(!miniClusterRunning, "Cluster has already been started"); if (externalZkConnectString == null) { - util.startMiniZKCluster(); + util.startMiniZKCluster(option.getNumZkServers()); } else { Configuration conf = util.getConfiguration(); conf.set(HConstants.ZOOKEEPER_QUORUM, externalZkConnectString); diff --git a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java index 81f9d279a226..e4a5083cb86b 100644 --- a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java +++ b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/testing/TestTestingHBaseCluster.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.List; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -67,7 +68,7 @@ public static void setUpBeforeClass() throws Exception { final Configuration conf = HBaseConfiguration.create(); conf.setInt(HConstants.MASTER_INFO_PORT, 0); CLUSTER = TestingHBaseCluster.create(TestingHBaseClusterOption.builder().numMasters(2) - .numRegionServers(3).numDataNodes(3).conf(conf).build()); + .numZkServers(3).numRegionServers(3).numDataNodes(3).conf(conf).build()); } @AfterClass @@ -162,15 +163,17 @@ public void testGetAddresses() throws Exception { final List masterAddrs = CLUSTER.getMasterAddresses(); assertEquals(2, masterAddrs.size()); assertTrue(masterAddrs.stream().allMatch(addr -> addr.matches(".*:[0-9]+$"))); - } - @Test - public void testGetPorts() throws Exception { + // ZooKeeper quorum: localhost:PORT1,localhost:PORT2,localhost:PORT3 + final String[] quorum = CLUSTER.getZooKeeperQuorum().orElse("").split(","); + assertEquals(3, quorum.length); + assertTrue(Stream.of(quorum).allMatch(a -> Pattern.matches(".*:[0-9]+$", a))); + + // Web UI addresses final String addressPattern = "^https?://.*:[0-9]+$"; assertTrue(CLUSTER.getActiveMasterInfoAddress().map(a -> Pattern.matches(addressPattern, a)) .orElse(false)); assertTrue(CLUSTER.getActiveNameNodeInfoAddress().map(a -> Pattern.matches(addressPattern, a)) .orElse(false)); - assertTrue(CLUSTER.getZooKeeperQuorum().isPresent()); } }